36#include "llvm/IR/IntrinsicsRISCV.h"
48#define DEBUG_TYPE "riscv-lower"
54 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
55 "instructions that we will consider for VW expansion"),
60 cl::desc(
"Allow the formation of VW_W operations (e.g., "
61 "VWADD_W) with splat constants"),
66 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
67 "transformation to multiplications by the reciprocal"),
72 cl::desc(
"Give the maximum number of instructions that we will "
73 "use for creating a floating-point immediate value"),
80 if (Subtarget.isRVE())
87 !Subtarget.hasStdExtF()) {
88 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
89 "doesn't support the F instruction set extension (ignoring "
93 !Subtarget.hasStdExtD()) {
94 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
95 "doesn't support the D instruction set extension (ignoring "
119 if (Subtarget.hasStdExtZfbfmin())
121 if (Subtarget.hasStdExtF())
123 if (Subtarget.hasStdExtD())
127 if (Subtarget.hasStdExtZfinx())
129 if (Subtarget.hasStdExtZdinx()) {
137 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
138 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
140 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
141 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
142 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
143 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
144 MVT::nxv4i64, MVT::nxv8i64};
146 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
147 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
149 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
151 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
154 auto addRegClassForRVV = [
this](
MVT VT) {
158 if (VT.getVectorMinNumElements() < MinElts)
161 unsigned Size = VT.getSizeInBits().getKnownMinValue();
164 RC = &RISCV::VRRegClass;
166 RC = &RISCV::VRM2RegClass;
168 RC = &RISCV::VRM4RegClass;
170 RC = &RISCV::VRM8RegClass;
177 for (
MVT VT : BoolVecVTs)
178 addRegClassForRVV(VT);
179 for (
MVT VT : IntVecVTs) {
180 if (VT.getVectorElementType() == MVT::i64 &&
183 addRegClassForRVV(VT);
187 for (
MVT VT : F16VecVTs)
188 addRegClassForRVV(VT);
191 for (
MVT VT : F32VecVTs)
192 addRegClassForRVV(VT);
195 for (
MVT VT : F64VecVTs)
196 addRegClassForRVV(VT);
199 auto addRegClassForFixedVectors = [
this](
MVT VT) {
206 if (useRVVForFixedLengthVectorVT(VT))
207 addRegClassForFixedVectors(VT);
210 if (useRVVForFixedLengthVectorVT(VT))
211 addRegClassForFixedVectors(VT);
250 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
266 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
271 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul())
278 if (!Subtarget.hasStdExtM())
283 {MVT::i8, MVT::i16, MVT::i32},
Custom);
292 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
295 }
else if (Subtarget.hasVendorXTHeadBb()) {
306 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
307 Subtarget.hasVendorXTHeadBb())
314 if (Subtarget.hasStdExtZbb()) {
326 if (Subtarget.hasVendorXTHeadBb()) {
338 if (!Subtarget.hasVendorXTHeadCondMov())
341 static const unsigned FPLegalNodeTypes[] = {
342 ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT,
343 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
354 static const unsigned FPOpToExpand[] = {
355 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW,
358 static const unsigned FPRndMode[] = {
359 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
365 if (Subtarget.hasStdExtZfbfmin()) {
382 static const unsigned ZfhminPromoteOps[] = {
390 ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
411 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
412 ISD::FEXP2, ISD::FLOG, ISD::FLOG2, ISD::FLOG10},
427 if (!Subtarget.hasStdExtZfa())
449 if (Subtarget.hasStdExtZfa())
461 if (Subtarget.hasStdExtZfa()) {
530 if (Subtarget.hasStdExtZicbop()) {
534 if (Subtarget.hasStdExtA()) {
537 }
else if (Subtarget.hasForcedAtomics()) {
556 {MVT::i8, MVT::i16},
Custom);
567 static const unsigned IntegerVPOps[] = {
568 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
569 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
570 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
571 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
572 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
573 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
574 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
575 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
576 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
577 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
578 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
581 static const unsigned FloatingPointVPOps[] = {
582 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
583 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
584 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
585 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
586 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
587 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
588 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
589 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
590 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
591 ISD::VP_FRINT, ISD::VP_FNEARBYINT};
593 static const unsigned IntegerVecReduceOps[] = {
594 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
595 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
596 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
598 static const unsigned FloatingPointVecReduceOps[] = {
599 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
600 ISD::VECREDUCE_FMAX};
611 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
612 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
613 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
617 for (
MVT VT : BoolVecVTs) {
640 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
644 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
668 ISD::VP_TRUNCATE, ISD::VP_SETCC},
681 for (
MVT VT : IntVecVTs) {
692 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
739 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
740 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
764 if (Subtarget.hasStdExtZvbb()) {
768 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
775 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
784 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
806 const auto SetCommonVFPActions = [&](
MVT VT) {
822 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
848 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
849 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
880 const auto SetCommonVFPExtLoadTruncStoreActions =
882 for (
auto SmallVT : SmallerVTs) {
889 for (
MVT VT : F16VecVTs) {
892 SetCommonVFPActions(VT);
897 for (
MVT VT : F32VecVTs) {
900 SetCommonVFPActions(VT);
901 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
906 for (
MVT VT : F64VecVTs) {
909 SetCommonVFPActions(VT);
910 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
911 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
917 if (!useRVVForFixedLengthVectorVT(VT))
956 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
960 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
987 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1003 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT,
Custom);
1006 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1007 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1036 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1037 ISD::VECREDUCE_UMIN},
1058 if (!useRVVForFixedLengthVectorVT(VT))
1079 ISD::MGATHER, ISD::MSCATTER},
1083 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1084 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1090 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1097 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT},
1135 if (Subtarget.hasForcedAtomics()) {
1138 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1139 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1140 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1141 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1145 if (Subtarget.hasVendorXTHeadMemIdx()) {
1183 if (Subtarget.hasStdExtZbb())
1186 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1189 if (Subtarget.hasStdExtZbkb())
1201 if (Subtarget.hasVendorXTHeadMemPair())
1224MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1229bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1231 bool IsScalable)
const {
1238 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1258 unsigned Intrinsic)
const {
1259 auto &
DL =
I.getModule()->getDataLayout();
1261 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1262 bool IsUnitStrided) {
1264 Info.ptrVal =
I.getArgOperand(PtrOp);
1268 MemTy =
I.getArgOperand(0)->getType();
1271 MemTy =
I.getType();
1286 if (
I.getMetadata(LLVMContext::MD_nontemporal) !=
nullptr)
1290 switch (Intrinsic) {
1293 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1294 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1295 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1296 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1297 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1298 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1299 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1300 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1301 case Intrinsic::riscv_masked_cmpxchg_i32:
1303 Info.memVT = MVT::i32;
1304 Info.ptrVal =
I.getArgOperand(0);
1306 Info.align =
Align(4);
1310 case Intrinsic::riscv_masked_strided_load:
1311 return SetRVVLoadStoreInfo( 1,
false,
1313 case Intrinsic::riscv_masked_strided_store:
1314 return SetRVVLoadStoreInfo( 1,
true,
1316 case Intrinsic::riscv_seg2_load:
1317 case Intrinsic::riscv_seg3_load:
1318 case Intrinsic::riscv_seg4_load:
1319 case Intrinsic::riscv_seg5_load:
1320 case Intrinsic::riscv_seg6_load:
1321 case Intrinsic::riscv_seg7_load:
1322 case Intrinsic::riscv_seg8_load:
1323 return SetRVVLoadStoreInfo( 0,
false,
1325 case Intrinsic::riscv_seg2_store:
1326 case Intrinsic::riscv_seg3_store:
1327 case Intrinsic::riscv_seg4_store:
1328 case Intrinsic::riscv_seg5_store:
1329 case Intrinsic::riscv_seg6_store:
1330 case Intrinsic::riscv_seg7_store:
1331 case Intrinsic::riscv_seg8_store:
1333 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1336 case Intrinsic::riscv_vle:
1337 case Intrinsic::riscv_vle_mask:
1338 case Intrinsic::riscv_vleff:
1339 case Intrinsic::riscv_vleff_mask:
1340 return SetRVVLoadStoreInfo( 1,
1343 case Intrinsic::riscv_vse:
1344 case Intrinsic::riscv_vse_mask:
1345 return SetRVVLoadStoreInfo( 1,
1348 case Intrinsic::riscv_vlse:
1349 case Intrinsic::riscv_vlse_mask:
1350 case Intrinsic::riscv_vloxei:
1351 case Intrinsic::riscv_vloxei_mask:
1352 case Intrinsic::riscv_vluxei:
1353 case Intrinsic::riscv_vluxei_mask:
1354 return SetRVVLoadStoreInfo( 1,
1357 case Intrinsic::riscv_vsse:
1358 case Intrinsic::riscv_vsse_mask:
1359 case Intrinsic::riscv_vsoxei:
1360 case Intrinsic::riscv_vsoxei_mask:
1361 case Intrinsic::riscv_vsuxei:
1362 case Intrinsic::riscv_vsuxei_mask:
1363 return SetRVVLoadStoreInfo( 1,
1366 case Intrinsic::riscv_vlseg2:
1367 case Intrinsic::riscv_vlseg3:
1368 case Intrinsic::riscv_vlseg4:
1369 case Intrinsic::riscv_vlseg5:
1370 case Intrinsic::riscv_vlseg6:
1371 case Intrinsic::riscv_vlseg7:
1372 case Intrinsic::riscv_vlseg8:
1373 case Intrinsic::riscv_vlseg2ff:
1374 case Intrinsic::riscv_vlseg3ff:
1375 case Intrinsic::riscv_vlseg4ff:
1376 case Intrinsic::riscv_vlseg5ff:
1377 case Intrinsic::riscv_vlseg6ff:
1378 case Intrinsic::riscv_vlseg7ff:
1379 case Intrinsic::riscv_vlseg8ff:
1380 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1383 case Intrinsic::riscv_vlseg2_mask:
1384 case Intrinsic::riscv_vlseg3_mask:
1385 case Intrinsic::riscv_vlseg4_mask:
1386 case Intrinsic::riscv_vlseg5_mask:
1387 case Intrinsic::riscv_vlseg6_mask:
1388 case Intrinsic::riscv_vlseg7_mask:
1389 case Intrinsic::riscv_vlseg8_mask:
1390 case Intrinsic::riscv_vlseg2ff_mask:
1391 case Intrinsic::riscv_vlseg3ff_mask:
1392 case Intrinsic::riscv_vlseg4ff_mask:
1393 case Intrinsic::riscv_vlseg5ff_mask:
1394 case Intrinsic::riscv_vlseg6ff_mask:
1395 case Intrinsic::riscv_vlseg7ff_mask:
1396 case Intrinsic::riscv_vlseg8ff_mask:
1397 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1400 case Intrinsic::riscv_vlsseg2:
1401 case Intrinsic::riscv_vlsseg3:
1402 case Intrinsic::riscv_vlsseg4:
1403 case Intrinsic::riscv_vlsseg5:
1404 case Intrinsic::riscv_vlsseg6:
1405 case Intrinsic::riscv_vlsseg7:
1406 case Intrinsic::riscv_vlsseg8:
1407 case Intrinsic::riscv_vloxseg2:
1408 case Intrinsic::riscv_vloxseg3:
1409 case Intrinsic::riscv_vloxseg4:
1410 case Intrinsic::riscv_vloxseg5:
1411 case Intrinsic::riscv_vloxseg6:
1412 case Intrinsic::riscv_vloxseg7:
1413 case Intrinsic::riscv_vloxseg8:
1414 case Intrinsic::riscv_vluxseg2:
1415 case Intrinsic::riscv_vluxseg3:
1416 case Intrinsic::riscv_vluxseg4:
1417 case Intrinsic::riscv_vluxseg5:
1418 case Intrinsic::riscv_vluxseg6:
1419 case Intrinsic::riscv_vluxseg7:
1420 case Intrinsic::riscv_vluxseg8:
1421 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1424 case Intrinsic::riscv_vlsseg2_mask:
1425 case Intrinsic::riscv_vlsseg3_mask:
1426 case Intrinsic::riscv_vlsseg4_mask:
1427 case Intrinsic::riscv_vlsseg5_mask:
1428 case Intrinsic::riscv_vlsseg6_mask:
1429 case Intrinsic::riscv_vlsseg7_mask:
1430 case Intrinsic::riscv_vlsseg8_mask:
1431 case Intrinsic::riscv_vloxseg2_mask:
1432 case Intrinsic::riscv_vloxseg3_mask:
1433 case Intrinsic::riscv_vloxseg4_mask:
1434 case Intrinsic::riscv_vloxseg5_mask:
1435 case Intrinsic::riscv_vloxseg6_mask:
1436 case Intrinsic::riscv_vloxseg7_mask:
1437 case Intrinsic::riscv_vloxseg8_mask:
1438 case Intrinsic::riscv_vluxseg2_mask:
1439 case Intrinsic::riscv_vluxseg3_mask:
1440 case Intrinsic::riscv_vluxseg4_mask:
1441 case Intrinsic::riscv_vluxseg5_mask:
1442 case Intrinsic::riscv_vluxseg6_mask:
1443 case Intrinsic::riscv_vluxseg7_mask:
1444 case Intrinsic::riscv_vluxseg8_mask:
1445 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1448 case Intrinsic::riscv_vsseg2:
1449 case Intrinsic::riscv_vsseg3:
1450 case Intrinsic::riscv_vsseg4:
1451 case Intrinsic::riscv_vsseg5:
1452 case Intrinsic::riscv_vsseg6:
1453 case Intrinsic::riscv_vsseg7:
1454 case Intrinsic::riscv_vsseg8:
1455 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1458 case Intrinsic::riscv_vsseg2_mask:
1459 case Intrinsic::riscv_vsseg3_mask:
1460 case Intrinsic::riscv_vsseg4_mask:
1461 case Intrinsic::riscv_vsseg5_mask:
1462 case Intrinsic::riscv_vsseg6_mask:
1463 case Intrinsic::riscv_vsseg7_mask:
1464 case Intrinsic::riscv_vsseg8_mask:
1465 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1468 case Intrinsic::riscv_vssseg2:
1469 case Intrinsic::riscv_vssseg3:
1470 case Intrinsic::riscv_vssseg4:
1471 case Intrinsic::riscv_vssseg5:
1472 case Intrinsic::riscv_vssseg6:
1473 case Intrinsic::riscv_vssseg7:
1474 case Intrinsic::riscv_vssseg8:
1475 case Intrinsic::riscv_vsoxseg2:
1476 case Intrinsic::riscv_vsoxseg3:
1477 case Intrinsic::riscv_vsoxseg4:
1478 case Intrinsic::riscv_vsoxseg5:
1479 case Intrinsic::riscv_vsoxseg6:
1480 case Intrinsic::riscv_vsoxseg7:
1481 case Intrinsic::riscv_vsoxseg8:
1482 case Intrinsic::riscv_vsuxseg2:
1483 case Intrinsic::riscv_vsuxseg3:
1484 case Intrinsic::riscv_vsuxseg4:
1485 case Intrinsic::riscv_vsuxseg5:
1486 case Intrinsic::riscv_vsuxseg6:
1487 case Intrinsic::riscv_vsuxseg7:
1488 case Intrinsic::riscv_vsuxseg8:
1489 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1492 case Intrinsic::riscv_vssseg2_mask:
1493 case Intrinsic::riscv_vssseg3_mask:
1494 case Intrinsic::riscv_vssseg4_mask:
1495 case Intrinsic::riscv_vssseg5_mask:
1496 case Intrinsic::riscv_vssseg6_mask:
1497 case Intrinsic::riscv_vssseg7_mask:
1498 case Intrinsic::riscv_vssseg8_mask:
1499 case Intrinsic::riscv_vsoxseg2_mask:
1500 case Intrinsic::riscv_vsoxseg3_mask:
1501 case Intrinsic::riscv_vsoxseg4_mask:
1502 case Intrinsic::riscv_vsoxseg5_mask:
1503 case Intrinsic::riscv_vsoxseg6_mask:
1504 case Intrinsic::riscv_vsoxseg7_mask:
1505 case Intrinsic::riscv_vsoxseg8_mask:
1506 case Intrinsic::riscv_vsuxseg2_mask:
1507 case Intrinsic::riscv_vsuxseg3_mask:
1508 case Intrinsic::riscv_vsuxseg4_mask:
1509 case Intrinsic::riscv_vsuxseg5_mask:
1510 case Intrinsic::riscv_vsuxseg6_mask:
1511 case Intrinsic::riscv_vsuxseg7_mask:
1512 case Intrinsic::riscv_vsuxseg8_mask:
1513 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1565 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1567 return (SrcBits == 64 && DestBits == 32);
1578 return (SrcBits == 64 && DestBits == 32);
1586 EVT MemVT = LD->getMemoryVT();
1587 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1597 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1605 return Subtarget.hasStdExtZbb();
1609 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb();
1620 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1625 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1629 EVT VT =
Y.getValueType();
1635 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1641 if (Subtarget.hasStdExtZbs())
1642 return X.getValueType().isScalarInteger();
1645 if (Subtarget.hasVendorXTHeadBs())
1646 return C !=
nullptr;
1648 return C &&
C->getAPIntValue().ule(10);
1665 assert(Ty->isIntegerTy());
1667 unsigned BitSize = Ty->getIntegerBitWidth();
1668 if (BitSize > Subtarget.
getXLen())
1672 int64_t Val = Imm.getSExtValue();
1680 if (!Subtarget.enableUnalignedScalarMem())
1697 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1704 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1708 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1720 case Instruction::Add:
1721 case Instruction::Sub:
1722 case Instruction::Mul:
1723 case Instruction::And:
1724 case Instruction::Or:
1725 case Instruction::Xor:
1726 case Instruction::FAdd:
1727 case Instruction::FSub:
1728 case Instruction::FMul:
1729 case Instruction::FDiv:
1730 case Instruction::ICmp:
1731 case Instruction::FCmp:
1733 case Instruction::Shl:
1734 case Instruction::LShr:
1735 case Instruction::AShr:
1736 case Instruction::UDiv:
1737 case Instruction::SDiv:
1738 case Instruction::URem:
1739 case Instruction::SRem:
1740 return Operand == 1;
1758 switch (II->getIntrinsicID()) {
1759 case Intrinsic::fma:
1760 case Intrinsic::vp_fma:
1761 return Operand == 0 || Operand == 1;
1762 case Intrinsic::vp_shl:
1763 case Intrinsic::vp_lshr:
1764 case Intrinsic::vp_ashr:
1765 case Intrinsic::vp_udiv:
1766 case Intrinsic::vp_sdiv:
1767 case Intrinsic::vp_urem:
1768 case Intrinsic::vp_srem:
1769 return Operand == 1;
1771 case Intrinsic::vp_add:
1772 case Intrinsic::vp_mul:
1773 case Intrinsic::vp_and:
1774 case Intrinsic::vp_or:
1775 case Intrinsic::vp_xor:
1776 case Intrinsic::vp_fadd:
1777 case Intrinsic::vp_fmul:
1778 case Intrinsic::vp_icmp:
1779 case Intrinsic::vp_fcmp:
1781 case Intrinsic::vp_sub:
1782 case Intrinsic::vp_fsub:
1783 case Intrinsic::vp_fdiv:
1784 return Operand == 0 || Operand == 1;
1800 for (
auto OpIdx :
enumerate(
I->operands())) {
1806 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
1820 for (
Use &U :
Op->uses()) {
1863 if (!Subtarget.hasStdExtZfa())
1866 bool IsSupportedVT =
false;
1867 if (VT == MVT::f16) {
1868 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
1869 }
else if (VT == MVT::f32) {
1870 IsSupportedVT =
true;
1871 }
else if (VT == MVT::f64) {
1872 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
1873 IsSupportedVT =
true;
1883 bool ForCodeSize)
const {
1884 bool IsLegalVT =
false;
1887 else if (VT == MVT::f32)
1889 else if (VT == MVT::f64)
1902 return Imm.isZero();
1905 int Cost = Imm.isNegZero()
1909 Subtarget.getFeatureBits());
1916 unsigned Index)
const {
1929 if ((ResElts * 2) != SrcElts)
1980 LHS.getOpcode() ==
ISD::AND && LHS.hasOneUse() &&
1982 uint64_t Mask = LHS.getConstantOperandVal(1);
1987 ShAmt = LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
1992 LHS = LHS.getOperand(0);
2001 int64_t
C = RHSC->getSExtValue();
2043 switch (KnownSize) {
2071 return RISCV::VRRegClassID;
2073 return RISCV::VRM2RegClassID;
2075 return RISCV::VRM4RegClassID;
2077 return RISCV::VRM8RegClassID;
2087 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2088 "Unexpected subreg numbering");
2089 return RISCV::sub_vrm1_0 +
Index;
2092 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2093 "Unexpected subreg numbering");
2094 return RISCV::sub_vrm2_0 +
Index;
2097 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2098 "Unexpected subreg numbering");
2099 return RISCV::sub_vrm4_0 +
Index;
2106 return RISCV::VRRegClassID;
2115std::pair<unsigned, unsigned>
2117 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2119 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2120 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2121 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2122 "Register classes not ordered");
2131 unsigned SubRegIdx = RISCV::NoSubRegister;
2132 for (
const unsigned RCID :
2133 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2134 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2138 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2143 return {SubRegIdx, InsertExtractIdx};
2148bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2177unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2184 "Unexpected opcode");
2186 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2188 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2191 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2261bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2262 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2271 "Expected legal fixed length vector!");
2274 unsigned MaxELen = Subtarget.
getELEN();
2307 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2314 "Expected to convert into a scalable vector!");
2315 assert(V.getValueType().isFixedLengthVector() &&
2316 "Expected a fixed length vector operand!");
2326 "Expected to convert into a fixed length vector!");
2327 assert(V.getValueType().isScalableVector() &&
2328 "Expected a scalable vector operand!");
2356static std::pair<SDValue, SDValue>
2369static std::pair<SDValue, SDValue>
2383static std::pair<SDValue, SDValue>
2406 EVT VT,
unsigned DefinedValues)
const {
2417 MVT DstVT =
Op.getSimpleValueType();
2424 if (Src.getSimpleValueType() == MVT::f16 &&
2432 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2440 Opc,
DL, DstVT, Src,
2454 MVT SrcVT = Src.getSimpleValueType();
2460 if (SatVT != DstEltVT)
2464 if (SrcEltSize > (2 * DstEltSize))
2467 MVT DstContainerVT = DstVT;
2468 MVT SrcContainerVT = SrcVT;
2474 "Expected same element count");
2483 {Src, Src, DAG.getCondCode(ISD::SETNE),
2484 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2488 if (DstEltSize > (2 * SrcEltSize)) {
2512 case ISD::FROUNDEVEN:
2514 case ISD::VP_FROUNDEVEN:
2518 case ISD::VP_FROUNDTOZERO:
2522 case ISD::VP_FFLOOR:
2530 case ISD::VP_FROUND:
2546 MVT VT =
Op.getSimpleValueType();
2553 MVT ContainerVT = VT;
2560 if (
Op->isVPOpcode()) {
2561 Mask =
Op.getOperand(1);
2565 VL =
Op.getOperand(2);
2587 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
2601 switch (
Op.getOpcode()) {
2607 case ISD::VP_FFLOOR:
2609 case ISD::FROUNDEVEN:
2610 case ISD::VP_FROUND:
2611 case ISD::VP_FROUNDEVEN:
2612 case ISD::VP_FROUNDTOZERO: {
2627 case ISD::FNEARBYINT:
2628 case ISD::VP_FNEARBYINT:
2641 Src, Src, Mask, VL);
2656 MVT VT =
Op.getSimpleValueType();
2660 MVT ContainerVT = VT;
2672 MVT MaskVT = Mask.getSimpleValueType();
2675 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
2676 DAG.getUNDEF(MaskVT), Mask, VL});
2680 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
2681 Chain = Src.getValue(1);
2697 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
2709 switch (
Op.getOpcode()) {
2720 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
2726 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
2730 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
2739 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
2740 Truncated, Mask, VL);
2746 Src, Src, Mask, VL);
2756 MVT VT =
Op.getSimpleValueType();
2786 if (
Merge.isUndef())
2798 if (
Merge.isUndef())
2825 return std::nullopt;
2842 bool IsInteger =
Op.getValueType().isInteger();
2844 std::optional<unsigned> SeqStepDenom;
2845 std::optional<int64_t> SeqStepNum, SeqAddend;
2846 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
2847 unsigned EltSizeInBits =
Op.getValueType().getScalarSizeInBits();
2848 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
2851 if (
Op.getOperand(Idx).isUndef())
2858 return std::nullopt;
2859 Val =
Op.getConstantOperandVal(Idx) &
2864 return std::nullopt;
2868 Val = *ExactInteger;
2870 return std::nullopt;
2876 unsigned IdxDiff = Idx - PrevElt->second;
2877 int64_t ValDiff =
SignExtend64(Val - PrevElt->first, EltSizeInBits);
2885 int64_t Remainder = ValDiff % IdxDiff;
2887 if (Remainder != ValDiff) {
2890 return std::nullopt;
2896 SeqStepNum = ValDiff;
2897 else if (ValDiff != SeqStepNum)
2898 return std::nullopt;
2901 SeqStepDenom = IdxDiff;
2902 else if (IdxDiff != *SeqStepDenom)
2903 return std::nullopt;
2907 if (!PrevElt || PrevElt->first != Val)
2908 PrevElt = std::make_pair(Val, Idx);
2912 if (!SeqStepNum || !SeqStepDenom)
2913 return std::nullopt;
2917 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
2918 if (
Op.getOperand(Idx).isUndef())
2922 Val =
Op.getConstantOperandVal(Idx) &
2930 (int64_t)(Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
2931 int64_t Addend =
SignExtend64(Val - ExpectedVal, EltSizeInBits);
2934 else if (Addend != SeqAddend)
2935 return std::nullopt;
2938 assert(SeqAddend &&
"Must have an addend if we have a step");
2940 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
2961 MVT ContainerVT = VT;
2970 Idx, DAG.
getUNDEF(ContainerVT), Mask, VL);
2980 MVT VT =
Op.getSimpleValueType();
3010 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3011 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELEN());
3020 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3021 MVT IntegerViaVecVT =
3026 unsigned BitPos = 0, IntegerEltIdx = 0;
3029 for (
unsigned I = 0;
I < NumElts;) {
3032 Bits |= ((
uint64_t)BitValue << BitPos);
3038 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3039 if (NumViaIntegerBits <= 32)
3042 Elts[IntegerEltIdx] = Elt;
3051 if (NumElts < NumViaIntegerBits) {
3055 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3080 "Unexpected type for i1 splat value");
3108 int64_t StepNumerator = SimpleVID->StepNumerator;
3109 unsigned StepDenominator = SimpleVID->StepDenominator;
3110 int64_t Addend = SimpleVID->Addend;
3112 assert(StepNumerator != 0 &&
"Invalid step");
3113 bool Negate =
false;
3114 int64_t SplatStepVal = StepNumerator;
3118 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3120 Negate = StepNumerator < 0;
3122 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3132 (SplatStepVal >= 0 || StepDenominator == 1) &&
isInt<5>(Addend)) {
3135 MVT VIDContainerVT =
3143 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3144 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3147 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3149 if (StepDenominator != 1) {
3154 if (Addend != 0 || Negate) {
3178 if (VT.
isInteger() && EltBitSize < 64 &&
3180 BV->getRepeatedSequence(Sequence) &&
3181 (Sequence.size() * EltBitSize) <= 64) {
3182 unsigned SeqLen = Sequence.size();
3185 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3186 ViaIntVT == MVT::i64) &&
3187 "Unexpected sequence type");
3189 unsigned EltIdx = 0;
3194 for (
const auto &SeqV : Sequence) {
3195 if (!SeqV.isUndef())
3197 << (EltIdx * EltBitSize));
3203 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3210 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3211 "Unexpected bitcast sequence");
3215 MVT ViaContainerVT =
3235 unsigned MostCommonCount = 0;
3237 unsigned NumUndefElts =
3245 unsigned NumScalarLoads = 0;
3251 ValueCounts.
insert(std::make_pair(V, 0));
3252 unsigned &
Count = ValueCounts[V];
3255 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3260 if (++
Count >= MostCommonCount) {
3262 MostCommonCount =
Count;
3266 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3267 unsigned NumDefElts = NumElts - NumUndefElts;
3268 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3274 ((MostCommonCount > DominantValueCountThreshold) ||
3282 const SDValue &V = OpIdx.value();
3283 if (V.isUndef() || !Processed.insert(V).second)
3285 if (ValueCounts[V] == 1) {
3294 return DAG.getConstant(V == V1, DL, XLenVT);
3314 "Illegal type which will result in reserved encoding");
3319 unsigned UndefCount = 0;
3328 Vec,
Offset, Mask, VL, Policy);
3339 Vec,
Offset, Mask, VL, Policy);
3354 if ((LoC >> 31) == HiC)
3366 return DAG.
getNode(ISD::BITCAST,
DL, VT, InterVec);
3381 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
3393 bool HasPassthru = Passthru && !Passthru.
isUndef();
3394 if (!HasPassthru && !Passthru)
3406 if (Scalar.getValueType().bitsLE(XLenVT)) {
3413 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
3423 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
3424 "Unexpected scalar for splat lowering!");
3436 "Unexpected vector MVT");
3456 auto InnerVT = VT.
bitsLE(M1VT) ? VT : M1VT;
3458 DAG.
getUNDEF(InnerVT), Scalar, VL);
3469 if (!Scalar.getValueType().bitsLE(XLenVT))
3472 VT,
DL, DAG, Subtarget);
3480 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
3491 auto InnerVT = VT.
bitsLE(M1VT) ? VT : M1VT;
3493 DAG.
getUNDEF(InnerVT), Scalar, VL);
3522 if (Src != V2.getOperand(0))
3526 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
3531 V2.getConstantOperandVal(1) != Mask.size())
3535 if (Mask[0] != 0 && Mask[0] != 1)
3540 for (
unsigned i = 1; i != Mask.size(); ++i)
3541 if (Mask[i] != Mask[i - 1] + 2)
3559 int Size = Mask.size();
3561 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
3567 EvenSrc = StartIndexes[0];
3568 OddSrc = StartIndexes[1];
3571 if (EvenSrc != 0 && OddSrc != 0)
3581 int HalfNumElts = NumElts / 2;
3582 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
3598 int Size = Mask.size();
3610 for (
int i = 0; i !=
Size; ++i) {
3616 int StartIdx = i - (M %
Size);
3624 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
3627 Rotation = CandidateRotation;
3628 else if (Rotation != CandidateRotation)
3633 int MaskSrc = M <
Size ? 0 : 1;
3638 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
3643 TargetSrc = MaskSrc;
3644 else if (TargetSrc != MaskSrc)
3651 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
3652 assert((LoSrc >= 0 || HiSrc >= 0) &&
3653 "Failed to find a rotated input vector!");
3668 MVT ContainerVT = VT;
3671 assert(Src.getSimpleValueType().isFixedLengthVector());
3675 MVT SrcContainerVT =
3688 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
3695 unsigned Shift = EvenElts ? 0 : EltBits;
3701 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
3727 auto findNonEXTRACT_SUBVECTORParent =
3728 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
3733 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
3734 Offset += Parent.getConstantOperandVal(1);
3735 Parent = Parent.getOperand(0);
3737 return std::make_pair(Parent,
Offset);
3740 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
3741 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
3750 for (
size_t i = 0; i != NewMask.
size(); ++i) {
3751 if (NewMask[i] == -1)
3754 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
3755 NewMask[i] = NewMask[i] + V1IndexOffset;
3759 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
3765 if (NewMask[0] <= 0)
3769 for (
unsigned i = 1; i != NewMask.
size(); ++i)
3770 if (NewMask[i - 1] + 1 != NewMask[i])
3774 MVT SrcVT = Src.getSimpleValueType();
3805 int NumSubElts,
Index;
3810 bool OpsSwapped = Mask[
Index] < (int)NumElts;
3811 SDValue InPlace = OpsSwapped ? V2 : V1;
3812 SDValue ToInsert = OpsSwapped ? V1 : V2;
3822 if (NumSubElts +
Index >= (
int)NumElts)
3836 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
3848 bool OpsSwapped =
false;
3863 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
3864 for (
unsigned i = S; i !=
E; ++i)
3865 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
3871 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
3872 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
3875 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
3877 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
3882 auto OpCode = IsVSlidedown ?
3885 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
3888 Splat, TrueMask, VL);
3899 MVT VecContainerVT = VecVT;
3916 MVT WideContainerVT = WideVT;
3922 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
3931 EvenV, OddV, Passthru, Mask, VL);
3937 AllOnesVec, Passthru, Mask, VL);
3945 OddsMul, Passthru, Mask, VL);
3951 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
3970 MVT VT =
Op.getSimpleValueType();
3978 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4002 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4003 V = V.getOperand(
Offset / OpElements);
4019 SDValue Ops[] = {Ld->getChain(),
4037 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
4038 Ld->getPointerInfo().getWithOffset(
Offset),
4039 Ld->getOriginalAlign(),
4043 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
4044 Ld->getOriginalAlign(),
4045 Ld->getMemOperand()->getFlags());
4056 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
4059 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
4081 LoV = LoSrc == 0 ? V1 : V2;
4085 HiV = HiSrc == 0 ? V1 : V2;
4091 unsigned InvRotate = NumElts - Rotation;
4101 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
4121 int EvenSrc, OddSrc;
4126 int Size = Mask.size();
4128 assert(EvenSrc >= 0 &&
"Undef source?");
4129 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
4133 assert(OddSrc >= 0 &&
"Undef source?");
4134 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
4145 int MaskIndex = MaskIdx.value();
4146 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
4149 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
4161 bool InvertMask = IsSelect == SwapOps;
4170 for (
int MaskIndex : Mask) {
4171 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
4174 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
4175 GatherIndicesLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
4179 IsLHSOrUndefIndex ? DAG.
getUNDEF(XLenVT)
4181 if (IsLHSOrUndefIndex && MaskIndex >= 0)
4182 ++LHSIndexCounts[MaskIndex];
4183 if (!IsLHSOrUndefIndex)
4184 ++RHSIndexCounts[MaskIndex - NumElts];
4190 std::swap(GatherIndicesLHS, GatherIndicesRHS);
4193 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
4219 MVT IndexContainerVT =
4233 if (LHSIndexCounts.
size() == 1) {
4234 int SplatIndex = LHSIndexCounts.
begin()->getFirst();
4235 Gather = DAG.
getNode(GatherVXOpc,
DL, ContainerVT, V1,
4237 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
4243 Gather = DAG.
getNode(GatherVVOpc,
DL, ContainerVT, V1, LHSIndices,
4244 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
4250 if (!V2.isUndef()) {
4260 if (RHSIndexCounts.
size() == 1) {
4261 int SplatIndex = RHSIndexCounts.
begin()->getFirst();
4262 Gather = DAG.
getNode(GatherVXOpc,
DL, ContainerVT, V2,
4269 Gather = DAG.
getNode(GatherVVOpc,
DL, ContainerVT, V2, RHSIndices, Gather,
4300RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
4302 MVT VT =
Op.getSimpleValueType();
4306 MVT ContainerVT = VT;
4309 if (
Op->isVPOpcode()) {
4310 Mask =
Op.getOperand(1);
4314 VL =
Op.getOperand(2);
4320 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 :
MVT::f32;
4322 FloatEltVT = MVT::f32;
4329 "Expected legal float type!");
4336 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
4339 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
4344 if (FloatVT.
bitsGT(VT)) {
4345 if (
Op->isVPOpcode())
4346 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
4355 if (!
Op->isVPOpcode())
4359 MVT ContainerFloatVT =
4362 Src, Mask, RTZRM, VL);
4369 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
4373 if (
Op->isVPOpcode()) {
4382 else if (IntVT.
bitsGT(VT))
4387 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
4392 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
4393 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
4398 unsigned Adjust = ExponentBias + (EltSize - 1);
4400 if (
Op->isVPOpcode())
4410 else if (
Op.getOpcode() == ISD::VP_CTLZ)
4411 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
4423 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
4426 Load->getMemoryVT(),
4427 *
Load->getMemOperand()))
4431 MVT VT =
Op.getSimpleValueType();
4433 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
4434 "Unexpected unaligned RVV load type");
4438 "Expecting equally-sized RVV vector types to be legal");
4440 Load->getPointerInfo(),
Load->getOriginalAlign(),
4441 Load->getMemOperand()->getFlags());
4452 assert(Store &&
Store->getValue().getValueType().isVector() &&
4453 "Expected vector store");
4456 Store->getMemoryVT(),
4457 *
Store->getMemOperand()))
4464 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
4465 "Unexpected unaligned RVV store type");
4469 "Expecting equally-sized RVV vector types to be legal");
4470 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
4472 Store->getPointerInfo(),
Store->getOriginalAlign(),
4473 Store->getMemOperand()->getFlags());
4478 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
4504 if (LoVal == HiVal) {
4524 if (Subtarget.hasStdExtZtso()) {
4532 return DAG.
getNode(ISD::MEMBARRIER, dl, MVT::Other,
Op.getOperand(0));
4540 return DAG.
getNode(ISD::MEMBARRIER, dl, MVT::Other,
Op.getOperand(0));
4548 MVT VT =
Op.getSimpleValueType();
4551 unsigned Check = CNode->getZExtValue();
4552 unsigned TDCMask = 0;
4580 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
4586 VL,
Op->getFlags());
4605 Mask, VL,
Op->getFlags());
4608 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
4617 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
4621 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
4640 EVT VT =
Op.getValueType();
4660 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
4665#define OP_CASE(NODE) \
4667 return RISCVISD::NODE##_VL;
4668 switch (
Op.getOpcode()) {
4713 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
4717 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
4721 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
4731 "not a RISC-V target specific op");
4733 "adding target specific op should update this function");
4749 "not a RISC-V target specific op");
4751 "adding target specific op should update this function");
4764 switch (
Op.getOpcode()) {
4767 case ISD::ATOMIC_FENCE:
4770 return lowerGlobalAddress(
Op, DAG);
4772 return lowerBlockAddress(
Op, DAG);
4774 return lowerConstantPool(
Op, DAG);
4776 return lowerJumpTable(
Op, DAG);
4778 return lowerGlobalTLSAddress(
Op, DAG);
4782 return lowerSELECT(
Op, DAG);
4784 return lowerBRCOND(
Op, DAG);
4786 return lowerVASTART(
Op, DAG);
4788 return lowerFRAMEADDR(
Op, DAG);
4790 return lowerRETURNADDR(
Op, DAG);
4792 return lowerShiftLeftParts(
Op, DAG);
4794 return lowerShiftRightParts(
Op, DAG,
true);
4796 return lowerShiftRightParts(
Op, DAG,
false);
4799 assert(Subtarget.hasVendorXTHeadBb() &&
4800 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
4801 "Unexpected custom legalization");
4806 case ISD::BITCAST: {
4808 EVT VT =
Op.getValueType();
4812 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
4818 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
4819 Subtarget.hasStdExtZfbfmin()) {
4824 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
4831 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32 &&
4832 Subtarget.hasStdExtZfa()) {
4849 "Unexpected types");
4883 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
4885 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
4887 return LowerINTRINSIC_VOID(
Op, DAG);
4889 return LowerIS_FPCLASS(
Op, DAG);
4891 MVT VT =
Op.getSimpleValueType();
4893 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
4901 if (!
Op.getSimpleValueType().isVector())
4903 return lowerVectorTruncLike(
Op, DAG);
4906 if (
Op.getOperand(0).getValueType().isVector() &&
4907 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
4908 return lowerVectorMaskExt(
Op, DAG, 1);
4911 if (
Op.getOperand(0).getValueType().isVector() &&
4912 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
4913 return lowerVectorMaskExt(
Op, DAG, -1);
4916 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
4918 return lowerINSERT_VECTOR_ELT(
Op, DAG);
4920 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
4922 MVT VT =
Op.getSimpleValueType();
4930 MVT ContainerVT = VT;
4935 DAG.
getUNDEF(ContainerVT), Scalar, VL);
4941 MVT VT =
Op.getSimpleValueType();
4976 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
4977 Op.getOperand(1).getValueType() == MVT::i32) {
4981 DAG.
getNode(ISD::FPOWI,
DL, MVT::f32, Op0,
Op.getOperand(1));
4990 case ISD::FP_EXTEND: {
4992 EVT VT =
Op.getValueType();
4995 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
4997 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
5000 return DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f64, FloatVal);
5003 if (!
Op.getValueType().isVector())
5005 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
5009 EVT VT =
Op.getValueType();
5012 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
5014 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
5022 if (!
Op.getValueType().isVector())
5024 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
5028 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
5040 MVT VT =
Op.getSimpleValueType();
5044 bool IsStrict =
Op->isStrictFPOpcode();
5045 SDValue Src =
Op.getOperand(0 + IsStrict);
5047 MVT SrcVT = Src.getSimpleValueType();
5052 "Unexpected vector element types");
5056 if (EltSize > (2 * SrcEltSize)) {
5068 Op.getOperand(0), Ext);
5072 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
5077 auto [FExt, Chain] =
5079 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
5086 if (SrcEltSize > (2 * EltSize)) {
5089 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
5094 Op.getOperand(0), Src);
5109 Op.getOperand(0), Src);
5123 unsigned RVVOpc = 0;
5124 switch (
Op.getOpcode()) {
5156 "Expected same element count");
5163 Op.getOperand(0), Src, Mask, VL);
5167 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
5173 case ISD::FP_TO_BF16: {
5182 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
5187 case ISD::BF16_TO_FP: {
5189 MVT VT =
Op.getSimpleValueType();
5199 return DAG.
getNode(ISD::FP_EXTEND,
DL, VT, Res);
5202 case ISD::FP_TO_FP16: {
5211 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
5216 case ISD::FP16_TO_FP: {
5234 case ISD::FNEARBYINT:
5237 case ISD::FROUNDEVEN:
5239 case ISD::VECREDUCE_ADD:
5240 case ISD::VECREDUCE_UMAX:
5241 case ISD::VECREDUCE_SMAX:
5242 case ISD::VECREDUCE_UMIN:
5243 case ISD::VECREDUCE_SMIN:
5244 return lowerVECREDUCE(
Op, DAG);
5245 case ISD::VECREDUCE_AND:
5246 case ISD::VECREDUCE_OR:
5247 case ISD::VECREDUCE_XOR:
5248 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
5249 return lowerVectorMaskVecReduction(
Op, DAG,
false);
5250 return lowerVECREDUCE(
Op, DAG);
5251 case ISD::VECREDUCE_FADD:
5252 case ISD::VECREDUCE_SEQ_FADD:
5253 case ISD::VECREDUCE_FMIN:
5254 case ISD::VECREDUCE_FMAX:
5255 return lowerFPVECREDUCE(
Op, DAG);
5256 case ISD::VP_REDUCE_ADD:
5257 case ISD::VP_REDUCE_UMAX:
5258 case ISD::VP_REDUCE_SMAX:
5259 case ISD::VP_REDUCE_UMIN:
5260 case ISD::VP_REDUCE_SMIN:
5261 case ISD::VP_REDUCE_FADD:
5262 case ISD::VP_REDUCE_SEQ_FADD:
5263 case ISD::VP_REDUCE_FMIN:
5264 case ISD::VP_REDUCE_FMAX:
5265 return lowerVPREDUCE(
Op, DAG);
5266 case ISD::VP_REDUCE_AND:
5267 case ISD::VP_REDUCE_OR:
5268 case ISD::VP_REDUCE_XOR:
5269 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
5270 return lowerVectorMaskVecReduction(
Op, DAG,
true);
5271 return lowerVPREDUCE(
Op, DAG);
5275 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
5278 return lowerINSERT_SUBVECTOR(
Op, DAG);
5280 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
5282 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
5284 return lowerVECTOR_INTERLEAVE(
Op, DAG);
5286 return lowerSTEP_VECTOR(
Op, DAG);
5288 return lowerVECTOR_REVERSE(
Op, DAG);
5290 return lowerVECTOR_SPLICE(
Op, DAG);
5294 if (
Op.getValueType().getVectorElementType() == MVT::i1)
5295 return lowerVectorMaskSplat(
Op, DAG);
5303 MVT VT =
Op.getSimpleValueType();
5304 unsigned NumOpElts =
5305 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
5308 SDValue SubVec = OpIdx.value();
5318 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
5320 if (
Op.getValueType().isFixedLengthVector())
5321 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
5324 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
5326 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
5327 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
5331 return lowerMaskedLoad(
Op, DAG);
5334 return lowerMaskedStore(
Op, DAG);
5343 EVT VT =
Op.getValueType();
5354 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
5356 MVT VT =
Op.getSimpleValueType();
5361 "Unexpected CondCode");
5387 return DAG.
getSetCC(
DL, VT, RHS, LHS, CCVal);
5390 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
5404 return lowerToScalableOp(
Op, DAG);
5408 if (
Op.getSimpleValueType().isFixedLengthVector())
5409 return lowerToScalableOp(
Op, DAG);
5411 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
5412 "Unexpected custom legalisation");
5432 return lowerToScalableOp(
Op, DAG);
5435 return lowerABS(
Op, DAG);
5439 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
5441 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
5443 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
5450 return lowerToScalableOp(
Op, DAG);
5453 return lowerVectorStrictFSetcc(
Op, DAG);
5463 case ISD::VP_GATHER:
5464 return lowerMaskedGather(
Op, DAG);
5466 case ISD::VP_SCATTER:
5467 return lowerMaskedScatter(
Op, DAG);
5469 return lowerGET_ROUNDING(
Op, DAG);
5470 case ISD::SET_ROUNDING:
5471 return lowerSET_ROUNDING(
Op, DAG);
5473 return lowerEH_DWARF_CFA(
Op, DAG);
5474 case ISD::VP_SELECT:
5520 case ISD::VP_FMINNUM:
5522 case ISD::VP_FMAXNUM:
5524 case ISD::VP_FCOPYSIGN:
5526 case ISD::VP_SIGN_EXTEND:
5527 case ISD::VP_ZERO_EXTEND:
5528 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
5529 return lowerVPExtMaskOp(
Op, DAG);
5530 return lowerVPOp(
Op, DAG,
5531 Op.getOpcode() == ISD::VP_SIGN_EXTEND
5534 case ISD::VP_TRUNCATE:
5535 return lowerVectorTruncLike(
Op, DAG);
5536 case ISD::VP_FP_EXTEND:
5537 case ISD::VP_FP_ROUND:
5538 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
5539 case ISD::VP_FP_TO_SINT:
5541 case ISD::VP_FP_TO_UINT:
5543 case ISD::VP_SINT_TO_FP:
5545 case ISD::VP_UINT_TO_FP:
5548 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
5549 return lowerVPSetCCMaskOp(
Op, DAG);
5559 case ISD::VP_BITREVERSE:
5564 case ISD::VP_CTLZ_ZERO_UNDEF:
5565 if (Subtarget.hasStdExtZvbb())
5567 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
5569 case ISD::VP_CTTZ_ZERO_UNDEF:
5570 if (Subtarget.hasStdExtZvbb())
5572 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
5575 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5576 return lowerVPStridedLoad(
Op, DAG);
5577 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5578 return lowerVPStridedStore(
Op, DAG);
5580 case ISD::VP_FFLOOR:
5582 case ISD::VP_FNEARBYINT:
5583 case ISD::VP_FROUND:
5584 case ISD::VP_FROUNDEVEN:
5585 case ISD::VP_FROUNDTOZERO:
5604 N->getOffset(), Flags);
5612template <
class NodeTy>
5614 bool IsLocal,
bool IsExternWeak)
const {
5624 if (IsLocal && !Subtarget.allowTaggedGlobals())
5638 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
5641 {DAG.getEntryNode(), Addr}, Ty,
MemOp);
5668 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
5672 {DAG.getEntryNode(), Addr}, Ty,
MemOp);
5687 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
5696 return getAddr(
N, DAG);
5703 return getAddr(
N, DAG);
5710 return getAddr(
N, DAG);
5715 bool UseGOT)
const {
5732 LLT(Ty.getSimpleVT()),
Align(Ty.getFixedSizeInBits() / 8));
5735 {DAG.getEntryNode(), Addr}, Ty,
MemOp);
5778 Args.push_back(Entry);
5794 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
5808 Addr = getStaticTLSAddr(
N, DAG,
false);
5811 Addr = getStaticTLSAddr(
N, DAG,
true);
5815 Addr = getDynamicTLSAddr(
N, DAG);
5832 if (LHS == LHS2 && RHS == RHS2) {
5837 }
else if (LHS == RHS2 && RHS == LHS2) {
5845 return std::nullopt;
5853 MVT VT =
N->getSimpleValueType(0);
5856 if (!Subtarget.hasShortForwardBranchOpt()) {
5892 if (std::optional<bool> MatchResult =
matchSetCC(LHS, RHS,
CC, TrueV)) {
5898 if (std::optional<bool> MatchResult =
matchSetCC(LHS, RHS,
CC, FalseV)) {
5915 "Unexpected condition code!");
5923 if (CCVal != ExpectedCCVal)
5929 if (!LHS.getValueType().isScalarInteger())
5939 int64_t CVal =
C->getSExtValue();
5966 if (Subtarget.hasShortForwardBranchOpt())
5969 unsigned SelOpNo = 0;
5979 unsigned ConstSelOpNo = 1;
5980 unsigned OtherSelOpNo = 2;
5987 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
5992 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
5998 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
6000 std::swap(NewConstOps[0], NewConstOps[1]);
6007 const APInt &NewConstAPInt =
6013 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
6015 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
6018 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
6019 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
6028 MVT VT =
Op.getSimpleValueType();
6042 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
6102 if (
Op.hasOneUse()) {
6103 unsigned UseOpc =
Op->use_begin()->getOpcode();
6109 return lowerSELECT(NewSel, DAG);
6137 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
6162 if (TrueVal - 1 == FalseVal)
6164 if (TrueVal + 1 == FalseVal)
6171 RHS == TrueV && LHS == FalseV) {
6214 LHS, RHS, TargetCC,
Op.getOperand(2));
6244 int XLenInBytes = Subtarget.
getXLen() / 8;
6246 EVT VT =
Op.getValueType();
6251 int Offset = -(XLenInBytes * 2);
6267 int XLenInBytes = Subtarget.
getXLen() / 8;
6272 EVT VT =
Op.getValueType();
6276 int Off = -XLenInBytes;
6277 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
6296 EVT VT =
Lo.getValueType();
6335 EVT VT =
Lo.getValueType();
6386 MVT VT =
Op.getSimpleValueType();
6399 "Unexpected type for i1 splat value");
6415 MVT VecVT =
Op.getSimpleValueType();
6417 "Unexpected SPLAT_VECTOR_PARTS lowering");
6438 if ((LoC >> 31) == HiC)
6446 Hi.getConstantOperandVal(1) == 31)
6461 int64_t ExtTrueVal)
const {
6463 MVT VecVT =
Op.getSimpleValueType();
6466 assert(Src.getValueType().isVector() &&
6467 Src.getValueType().getVectorElementType() == MVT::i1);
6488 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
6490 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
6492 SplatTrueVal, SplatZero, VL);
6497SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
6499 MVT ExtVT =
Op.getSimpleValueType();
6503 MVT VT =
Op.getOperand(0).getSimpleValueType();
6529 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
6531 EVT MaskVT =
Op.getValueType();
6534 "Unexpected type for vector mask lowering");
6536 MVT VecVT = Src.getSimpleValueType();
6540 VL =
Op.getOperand(2);
6543 MVT ContainerVT = VecVT;
6549 MVT MaskContainerVT =
6556 std::tie(Mask, VL) =
6564 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
6566 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
6570 DAG.
getUNDEF(ContainerVT), Mask, VL);
6581 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
6584 MVT VT =
Op.getSimpleValueType();
6586 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
6590 return lowerVectorMaskTruncLike(
Op, DAG);
6598 MVT SrcVT = Src.getSimpleValueType();
6603 "Unexpected vector truncate lowering");
6605 MVT ContainerVT = SrcVT;
6609 VL =
Op.getOperand(2);
6622 std::tie(Mask, VL) =
6633 }
while (SrcEltVT != DstEltVT);
6642RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
6647 MVT VT =
Op.getSimpleValueType();
6648 MVT SrcVT = Src.getSimpleValueType();
6649 MVT ContainerVT = VT;
6670 Chain, Src, Mask, VL);
6671 Chain = Src.getValue(1);
6678 Chain, Src, Mask, VL);
6689RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
6692 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
6694 Op.getOpcode() == ISD::VP_FP_EXTEND ||
Op.getOpcode() == ISD::FP_EXTEND;
6699 MVT VT =
Op.getSimpleValueType();
6701 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
6704 MVT SrcVT = Src.getSimpleValueType();
6711 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
6714 MVT ContainerVT = VT;
6718 VL =
Op.getOperand(2);
6732 std::tie(Mask, VL) =
6738 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
6744 unsigned InterConvOpc =
6749 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
6751 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
6766 MVT VecVT =
Op.getSimpleValueType();
6780 MVT ContainerVT = VecVT;
6797 IsLegalInsert =
true;
6806 if (IsLegalInsert) {
6810 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
6822 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
6823 MVT I32ContainerVT =
6834 Vec, Vec, ValLo, I32Mask, InsertI64VL);
6839 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
6841 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
6852 DAG.
getUNDEF(I32ContainerVT), ValLo,
6853 I32Mask, InsertI64VL);
6855 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
6856 I32Mask, InsertI64VL);
6858 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
6872 Idx, Mask, InsertVL, Policy);
6887 EVT EltVT =
Op.getValueType();
6894 MVT ContainerVT = VecVT;
6909 unsigned WidenVecLen;
6912 unsigned MaxEEW = Subtarget.
getELEN();
6917 "the number of elements should be power of 2");
6921 ExtractBitIdx = Idx;
6923 WideEltVT = LargestEltVT;
6926 ExtractElementIdx = DAG.
getNode(
6935 Vec = DAG.
getNode(ISD::BITCAST,
DL, WideVT, Vec);
6937 Vec, ExtractElementIdx);
6952 MVT ContainerVT = VecVT;
6963 DAG.
getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
6983 "Unexpected opcode");
6990 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
6995 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
7012 if (OpVT.
bitsLT(XLenVT)) {
7019 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
7030 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
7033 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
7044 case Intrinsic::riscv_vslide1up:
7045 case Intrinsic::riscv_vslide1down:
7046 case Intrinsic::riscv_vslide1up_mask:
7047 case Intrinsic::riscv_vslide1down_mask: {
7050 bool IsMasked = NumOps == 7;
7056 std::tie(ScalarLo, ScalarHi) =
7077 if (AVLInt <= MinVLMAX) {
7079 }
else if (AVLInt >= 2 * MaxVLMAX) {
7086 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
7120 if (IntNo == Intrinsic::riscv_vslide1up ||
7121 IntNo == Intrinsic::riscv_vslide1up_mask) {
7123 ScalarHi, I32Mask, I32VL);
7125 ScalarLo, I32Mask, I32VL);
7128 ScalarLo, I32Mask, I32VL);
7130 ScalarHi, I32Mask, I32VL);
7180 const unsigned ElementWidth = 8;
7187 unsigned VF =
N->getConstantOperandVal(2);
7192 bool Fractional = VF < LMul1VF;
7193 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
7210 unsigned IntNo =
Op.getConstantOperandVal(0);
7217 case Intrinsic::thread_pointer: {
7221 case Intrinsic::riscv_orc_b:
7222 case Intrinsic::riscv_brev8:
7223 case Intrinsic::riscv_sha256sig0:
7224 case Intrinsic::riscv_sha256sig1:
7225 case Intrinsic::riscv_sha256sum0:
7226 case Intrinsic::riscv_sha256sum1:
7227 case Intrinsic::riscv_sm3p0:
7228 case Intrinsic::riscv_sm3p1: {
7241 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
7243 case Intrinsic::riscv_sm4ks:
7244 case Intrinsic::riscv_sm4ed: {
7247 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
7250 case Intrinsic::riscv_zip:
7251 case Intrinsic::riscv_unzip: {
7254 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
7256 case Intrinsic::riscv_clmul:
7259 case Intrinsic::riscv_clmulh:
7262 case Intrinsic::riscv_clmulr:
7265 case Intrinsic::experimental_get_vector_length:
7267 case Intrinsic::riscv_vmv_x_s:
7268 assert(
Op.getValueType() == XLenVT &&
"Unexpected VT!");
7271 case Intrinsic::riscv_vfmv_f_s:
7274 case Intrinsic::riscv_vmv_v_x:
7276 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
7278 case Intrinsic::riscv_vfmv_v_f:
7280 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
7281 case Intrinsic::riscv_vmv_s_x: {
7284 if (
Scalar.getValueType().bitsLE(XLenVT)) {
7287 Op.getOperand(1), Scalar,
Op.getOperand(3));
7290 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
7307 MVT VT =
Op.getSimpleValueType();
7312 if (
Op.getOperand(1).isUndef())
7335 unsigned IntNo =
Op.getConstantOperandVal(1);
7339 case Intrinsic::riscv_masked_strided_load: {
7348 MVT VT =
Op->getSimpleValueType(0);
7349 MVT ContainerVT = VT;
7374 ScalarVT,
Load->getMemOperand());
7380 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
7385 Ops.push_back(DAG.
getUNDEF(ContainerVT));
7387 Ops.push_back(PassThru);
7389 Ops.push_back(Stride);
7391 Ops.push_back(Mask);
7396 Ops.push_back(Policy);
7402 Load->getMemoryVT(),
Load->getMemOperand());
7403 Chain =
Result.getValue(1);
7409 case Intrinsic::riscv_seg2_load:
7410 case Intrinsic::riscv_seg3_load:
7411 case Intrinsic::riscv_seg4_load:
7412 case Intrinsic::riscv_seg5_load:
7413 case Intrinsic::riscv_seg6_load:
7414 case Intrinsic::riscv_seg7_load:
7415 case Intrinsic::riscv_seg8_load: {
7418 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
7419 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
7420 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
7421 Intrinsic::riscv_vlseg8};
7422 unsigned NF =
Op->getNumValues() - 1;
7423 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
7425 MVT VT =
Op->getSimpleValueType(0);
7432 ContainerVTs.push_back(MVT::Other);
7440 Load->getMemoryVT(),
Load->getMemOperand());
7442 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
7455 unsigned IntNo =
Op.getConstantOperandVal(1);
7459 case Intrinsic::riscv_masked_strided_store: {
7470 MVT ContainerVT = VT;
7484 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
7497 Ops,
Store->getMemoryVT(),
7498 Store->getMemOperand());
7500 case Intrinsic::riscv_seg2_store:
7501 case Intrinsic::riscv_seg3_store:
7502 case Intrinsic::riscv_seg4_store:
7503 case Intrinsic::riscv_seg5_store:
7504 case Intrinsic::riscv_seg6_store:
7505 case Intrinsic::riscv_seg7_store:
7506 case Intrinsic::riscv_seg8_store: {
7509 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
7510 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
7511 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
7512 Intrinsic::riscv_vsseg8};
7515 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
7517 MVT VT =
Op->getOperand(2).getSimpleValueType();
7526 for (
unsigned i = 0; i < NF; i++)
7528 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
7533 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
7541 switch (ISDOpcode) {
7544 case ISD::VECREDUCE_ADD:
7546 case ISD::VECREDUCE_UMAX:
7548 case ISD::VECREDUCE_SMAX:
7550 case ISD::VECREDUCE_UMIN:
7552 case ISD::VECREDUCE_SMIN:
7554 case ISD::VECREDUCE_AND:
7556 case ISD::VECREDUCE_OR:
7558 case ISD::VECREDUCE_XOR:
7567 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
7569 assert((
Op.getOpcode() == ISD::VECREDUCE_AND ||
7570 Op.getOpcode() == ISD::VECREDUCE_OR ||
7571 Op.getOpcode() == ISD::VECREDUCE_XOR ||
7572 Op.getOpcode() == ISD::VP_REDUCE_AND ||
7573 Op.getOpcode() == ISD::VP_REDUCE_OR ||
7574 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
7575 "Unexpected reduction lowering");
7578 assert(
Op.getValueType() == XLenVT &&
7579 "Expected reduction output to be legalized to XLenVT");
7581 MVT ContainerVT = VecVT;
7590 VL =
Op.getOperand(3);
7592 std::tie(Mask, VL) =
7600 switch (
Op.getOpcode()) {
7603 case ISD::VECREDUCE_AND:
7604 case ISD::VP_REDUCE_AND: {
7613 case ISD::VECREDUCE_OR:
7614 case ISD::VP_REDUCE_OR:
7620 case ISD::VECREDUCE_XOR:
7621 case ISD::VP_REDUCE_XOR: {
7643 return DAG.
getNode(BaseOpc,
DL, XLenVT, SetCC,
Op.getOperand(0));
7649 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
7650 (ImmAVL && ImmAVL->getZExtValue() >= 1);
7666 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
7670 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
7673 if (M1VT != InnerVT)
7679 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
7698 VecEVT =
Lo.getValueType();
7711 MVT ContainerVT = VecVT;
7722 Mask, VL,
DL, DAG, Subtarget);
7728static std::tuple<unsigned, SDValue, SDValue>
7731 auto Flags =
Op->getFlags();
7732 unsigned Opcode =
Op.getOpcode();
7737 case ISD::VECREDUCE_FADD: {
7743 case ISD::VECREDUCE_SEQ_FADD:
7746 case ISD::VECREDUCE_FMIN:
7749 case ISD::VECREDUCE_FMAX:
7758 MVT VecEltVT =
Op.getSimpleValueType();
7762 std::tie(RVVOpcode, VectorVal, ScalarVal) =
7766 MVT ContainerVT = VecVT;
7774 VectorVal, Mask, VL,
DL, DAG, Subtarget);
7778 switch (ISDOpcode) {
7781 case ISD::VP_REDUCE_ADD:
7783 case ISD::VP_REDUCE_UMAX:
7785 case ISD::VP_REDUCE_SMAX:
7787 case ISD::VP_REDUCE_UMIN:
7789 case ISD::VP_REDUCE_SMIN:
7791 case ISD::VP_REDUCE_AND:
7793 case ISD::VP_REDUCE_OR:
7795 case ISD::VP_REDUCE_XOR:
7797 case ISD::VP_REDUCE_FADD:
7799 case ISD::VP_REDUCE_SEQ_FADD:
7801 case ISD::VP_REDUCE_FMAX:
7803 case ISD::VP_REDUCE_FMIN:
7830 Vec, Mask, VL,
DL, DAG, Subtarget);
7842 unsigned OrigIdx =
Op.getConstantOperandVal(2);
7851 (OrigIdx != 0 || !Vec.
isUndef())) {
7854 assert(OrigIdx % 8 == 0 &&
"Invalid index");
7857 "Unexpected mask vector lowering");
7890 MVT ContainerVT = VecVT;
7921 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
7922 SlideupAmt, Mask, VL, Policy);
7930 unsigned SubRegIdx, RemIdx;
7931 std::tie(SubRegIdx, RemIdx) =
7933 VecVT, SubVecVT, OrigIdx,
TRI);
7952 if (RemIdx == 0 && (!IsSubVecPartReg || Vec.
isUndef()))
7960 MVT InterSubVT = VecVT;
7962 unsigned AlignedIdx = OrigIdx - RemIdx;
7991 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
7992 SlideupAmt, Mask, VL);
7997 if (VecVT.
bitsGT(InterSubVT))
8003 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
8009 MVT SubVecVT =
Op.getSimpleValueType();
8014 unsigned OrigIdx =
Op.getConstantOperandVal(1);
8025 assert(OrigIdx % 8 == 0 &&
"Invalid index");
8028 "Unexpected mask vector lowering");
8064 MVT ContainerVT = VecVT;
8077 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
8084 unsigned SubRegIdx, RemIdx;
8085 std::tie(SubRegIdx, RemIdx) =
8087 VecVT, SubVecVT, OrigIdx,
TRI);
8101 MVT InterSubVT = VecVT;
8116 Vec, SlidedownAmt, Mask, VL);
8125 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
8132 MVT VT =
N.getSimpleValueType();
8136 assert(
Op.getSimpleValueType() == VT &&
8137 "Operands and result must be same type");
8141 unsigned NumVals =
N->getNumValues();
8144 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
8147 for (
unsigned I = 0;
I < NumVals;
I++) {
8153 if (TruncVals.
size() > 1)
8155 return TruncVals.
front();
8161 MVT VecVT =
Op.getSimpleValueType();
8165 "vector_interleave on non-scalable vector!");
8176 EVT SplitVT = Op0Lo.getValueType();
8179 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
8181 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
8195 Op.getOperand(0),
Op.getOperand(1));
8222 Concat, EvenIdx, Passthru, Mask, VL);
8224 Concat, OddIdx, Passthru, Mask, VL);
8238 MVT VecVT =
Op.getSimpleValueType();
8241 "vector_interleave on non-scalable vector!");
8254 EVT SplitVT = Op0Lo.getValueType();
8257 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
8259 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
8281 Op.getOperand(0),
Op.getOperand(1));
8329 MVT VT =
Op.getSimpleValueType();
8334 uint64_t StepValImm =
Op.getConstantOperandVal(0);
8335 if (StepValImm != 1) {
8344 VL, VT,
DL, DAG, Subtarget);
8359 MVT VecVT =
Op.getSimpleValueType();
8379 if (MaxVLMAX > 256 && EltSize == 8) {
8426 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
8436 MVT VecVT =
Op.getSimpleValueType();
8442 if (ImmValue >= 0) {
8458 DownOffset, TrueMask, UpOffset);
8459 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
8465RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
8471 Load->getMemoryVT(),
8472 *
Load->getMemOperand()) &&
8473 "Expecting a correctly-aligned load");
8475 MVT VT =
Op.getSimpleValueType();
8483 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
8492 Load->getMemoryVT(),
Load->getMemOperand());
8499RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
8505 Store->getMemoryVT(),
8506 *
Store->getMemOperand()) &&
8507 "Expecting a correctly-aligned store");
8530 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
8533 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
8534 Store->getMemoryVT(),
Store->getMemOperand());
8540 MVT VT =
Op.getSimpleValueType();
8543 EVT MemVT = MemSD->getMemoryVT();
8545 SDValue Chain = MemSD->getChain();
8550 Mask = VPLoad->getMask();
8552 VL = VPLoad->getVectorLength();
8555 Mask = MLoad->getMask();
8556 PassThru = MLoad->getPassThru();
8563 MVT ContainerVT = VT;
8577 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
8594 Chain =
Result.getValue(1);
8607 EVT MemVT = MemSD->getMemoryVT();
8609 SDValue Chain = MemSD->getChain();
8614 Val = VPStore->getValue();
8615 Mask = VPStore->getMask();
8616 VL = VPStore->getVectorLength();
8619 Val = MStore->getValue();
8620 Mask = MStore->getMask();
8628 MVT ContainerVT = VT;
8643 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
8652 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
8656RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
8658 MVT InVT =
Op.getOperand(0).getSimpleValueType();
8661 MVT VT =
Op.getSimpleValueType();
8682 unsigned Opc =
Op.getOpcode();
8689 MVT VT =
Op.getSimpleValueType();
8722 MVT ContainerInVT = InVT;
8741 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
8745 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
8753 {Chain, Op1, Op2, CC, Mask, Mask, VL});
8758 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
8771 MVT VT =
Op.getSimpleValueType();
8775 "Unexpected type for ISD::ABS");
8777 MVT ContainerVT = VT;
8784 if (
Op->getOpcode() == ISD::VP_ABS) {
8785 Mask =
Op->getOperand(1);
8789 VL =
Op->getOperand(2);
8797 DAG.
getUNDEF(ContainerVT), Mask, VL);
8799 DAG.
getUNDEF(ContainerVT), Mask, VL);
8806SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
8809 MVT VT =
Op.getSimpleValueType();
8813 "Can only handle COPYSIGN with matching types.");
8822 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
8827SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
8829 MVT VT =
Op.getSimpleValueType();
8857 MVT VT =
Op.getSimpleValueType();
8862 for (
const SDValue &V :
Op->op_values()) {
8866 if (!V.getValueType().isVector()) {
8872 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
8873 "Only fixed length vectors are supported!");
8887 if (
Op->isStrictFPOpcode()) {
8896 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
8906 unsigned RISCVISDOpc,
8907 bool HasMergeOp)
const {
8909 MVT VT =
Op.getSimpleValueType();
8912 MVT ContainerVT = VT;
8923 if (!V.getValueType().isFixedLengthVector()) {
8928 MVT OpVT = V.getSimpleValueType();
8930 assert(useRVVForFixedLengthVectorVT(OpVT) &&
8931 "Only fixed length vectors are supported!");
8936 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
8946 MVT VT =
Op.getSimpleValueType();
8952 MVT ContainerVT = VT;
8962 DAG.
getUNDEF(ContainerVT), Zero, VL);
8965 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
8967 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
8970 Splat, ZeroSplat, VL);
8979 MVT VT =
Op.getSimpleValueType();
8987 MVT ContainerVT = VT;
8997 switch (Condition) {
9057 unsigned RISCVISDOpc)
const {
9064 MVT DstVT =
Op.getSimpleValueType();
9065 MVT SrcVT = Src.getSimpleValueType();
9078 if (DstEltSize >= SrcEltSize) {
9087 if (SrcEltSize == 1) {
9099 }
else if (DstEltSize > (2 * SrcEltSize)) {
9103 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
9109 "Wrong input/output vector types");
9112 if (DstEltSize > (2 * SrcEltSize)) {
9128 MVT InterimFVT = DstVT;
9129 if (SrcEltSize > (2 * DstEltSize)) {
9130 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
9137 if (InterimFVT != DstVT) {
9143 "Wrong input/output vector types");
9147 if (DstEltSize == 1) {
9150 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
9160 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
9170 while (InterimIVT != DstVT) {
9182 MVT VT =
Op.getSimpleValueType();
9190 unsigned VecOpc)
const {
9191 MVT VT =
Op.getSimpleValueType();
9193 return lowerVPOp(
Op, DAG, VecOpc,
true);
9200 MVT ContainerVT = VT;
9219 MVT VT =
Op.getSimpleValueType();
9220 MVT ContainerVT = VT;
9232 : Intrinsic::riscv_vlse_mask,
9235 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
9236 VPNode->getStride()};
9244 Ops.
push_back(VPNode->getVectorLength());
9252 VPNode->getMemoryVT(), VPNode->getMemOperand());
9267 SDValue StoreVal = VPNode->getValue();
9269 MVT ContainerVT = VT;
9280 : Intrinsic::riscv_vsse_mask,
9283 VPNode->getBasePtr(), VPNode->getStride()};
9291 Ops.
push_back(VPNode->getVectorLength());
9294 Ops, VPNode->getMemoryVT(),
9295 VPNode->getMemOperand());
9307 MVT VT =
Op.getSimpleValueType();
9310 EVT MemVT = MemSD->getMemoryVT();
9312 SDValue Chain = MemSD->getChain();
9319 Index = VPGN->getIndex();
9320 Mask = VPGN->getMask();
9322 VL = VPGN->getVectorLength();
9328 Index = MGN->getIndex();
9329 Mask = MGN->getMask();
9330 PassThru = MGN->getPassThru();
9334 MVT IndexVT =
Index.getSimpleValueType();
9339 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
9342 "Unexpected extending MGATHER/VP_GATHER");
9349 MVT ContainerVT = VT;
9376 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
9393 Chain =
Result.getValue(1);
9411 EVT MemVT = MemSD->getMemoryVT();
9413 SDValue Chain = MemSD->getChain();
9416 bool IsTruncatingStore =
false;
9420 Index = VPSN->getIndex();
9421 Mask = VPSN->getMask();
9422 Val = VPSN->getValue();
9423 VL = VPSN->getVectorLength();
9425 IsTruncatingStore =
false;
9429 Index = MSN->getIndex();
9430 Mask = MSN->getMask();
9431 Val = MSN->getValue();
9432 IsTruncatingStore = MSN->isTruncatingStore();
9436 MVT IndexVT =
Index.getSimpleValueType();
9441 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
9444 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
9445 (void)IsTruncatingStore;
9451 MVT ContainerVT = VT;
9478 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
9488 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
9497 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
9505 static const int Table =
9529 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
9535 static const unsigned Table =
9556 bool isRISCV64 = Subtarget.
is64Bit();
9620 switch (
N->getOpcode()) {
9622 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
9628 "Unexpected custom legalisation");
9629 bool IsStrict =
N->isStrictFPOpcode();
9632 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
9650 Opc,
DL, VTs, Chain, Op0,
9658 Op0 = DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f32, Op0);
9681 std::tie(Result, Chain) =
9698 Op0 = DAG.
getNode(ISD::FP_EXTEND,
DL, MVT::f32, Op0);
9710 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
9719 case ISD::READCYCLECOUNTER: {
9721 "READCYCLECOUNTER only has custom type legalization on riscv32");
9749 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
9750 unsigned XLen = Subtarget.
getXLen();
9753 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
9761 if (LHSIsU == RHSIsU)
9778 if (RHSIsU && LHSIsS && !RHSIsS)
9779 Results.push_back(MakeMULPair(LHS, RHS));
9780 else if (LHSIsU && RHSIsS && !LHSIsS)
9781 Results.push_back(MakeMULPair(RHS, LHS));
9790 "Unexpected custom legalisation");
9797 "Unexpected custom legalisation");
9800 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
9826 "Unexpected custom legalisation");
9827 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
9828 Subtarget.hasVendorXTHeadBb()) &&
9829 "Unexpected custom legalization");
9831 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
9840 "Unexpected custom legalisation");
9854 MVT VT =
N->getSimpleValueType(0);
9855 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
9856 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
9857 "Unexpected custom legalisation");
9878 "Unexpected custom legalisation");
9899 EVT OType =
N->getValueType(1);
9912 "Unexpected custom legalisation");
9929 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
9933 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
9940 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res, LHS,
9951 "Unexpected custom legalisation");
9952 if (Subtarget.hasStdExtZbb()) {
9972 "Unexpected custom legalisation");
9974 if (Subtarget.hasStdExtZbb()) {
10007 case ISD::BITCAST: {
10008 EVT VT =
N->getValueType(0);
10013 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
10017 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
10018 Subtarget.hasStdExtZfbfmin()) {
10021 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
10026 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32 &&
10027 Subtarget.hasStdExtZfa()) {
10029 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
10049 MVT VT =
N->getSimpleValueType(0);
10051 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
10052 "Unexpected custom legalisation");
10053 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
10079 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
10081 "Unexpected EXTRACT_VECTOR_ELT legalization");
10084 MVT ContainerVT = VecVT;
10099 DAG.
getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
10112 DAG.
getUNDEF(ContainerVT), Mask, VL);
10124 "Don't know how to custom type legalize this intrinsic!");
10125 case Intrinsic::experimental_get_vector_length: {
10130 case Intrinsic::riscv_orc_b:
10131 case Intrinsic::riscv_brev8:
10132 case Intrinsic::riscv_sha256sig0:
10133 case Intrinsic::riscv_sha256sig1:
10134 case Intrinsic::riscv_sha256sum0:
10135 case Intrinsic::riscv_sha256sum1:
10136 case Intrinsic::riscv_sm3p0:
10137 case Intrinsic::riscv_sm3p1: {
10138 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
10158 case Intrinsic::riscv_sm4ks:
10159 case Intrinsic::riscv_sm4ed: {
10171 case Intrinsic::riscv_clmul: {
10172 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
10183 case Intrinsic::riscv_clmulh:
10184 case Intrinsic::riscv_clmulr: {
10185 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
10213 case Intrinsic::riscv_vmv_x_s: {
10214 EVT VT =
N->getValueType(0);
10216 if (VT.
bitsLT(XLenVT)) {
10225 "Unexpected custom legalization");
10252 case ISD::VECREDUCE_ADD:
10253 case ISD::VECREDUCE_AND:
10254 case ISD::VECREDUCE_OR:
10255 case ISD::VECREDUCE_XOR:
10256 case ISD::VECREDUCE_SMAX:
10257 case ISD::VECREDUCE_UMAX:
10258 case ISD::VECREDUCE_SMIN:
10259 case ISD::VECREDUCE_UMIN:
10263 case ISD::VP_REDUCE_ADD:
10264 case ISD::VP_REDUCE_AND:
10265 case ISD::VP_REDUCE_OR:
10266 case ISD::VP_REDUCE_XOR:
10267 case ISD::VP_REDUCE_SMAX:
10268 case ISD::VP_REDUCE_UMAX:
10269 case ISD::VP_REDUCE_SMIN:
10270 case ISD::VP_REDUCE_UMIN:
10287 auto BinOpToRVVReduce = [](
unsigned Opc) {
10316 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
10319 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
10322 unsigned Opc =
N->getOpcode();
10323 unsigned ReduceIdx;
10324 if (IsReduction(
N->getOperand(0), Opc))
10326 else if (IsReduction(
N->getOperand(1), Opc))
10332 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
10394 if (!Subtarget.hasStdExtZba())
10398 EVT VT =
N->getValueType(0);
10414 int64_t C0 = N0C->getSExtValue();
10415 int64_t C1 = N1C->getSExtValue();
10416 if (C0 <= 0 || C1 <= 0)
10420 int64_t Bits = std::min(C0, C1);
10421 int64_t Diff = std::abs(C0 - C1);
10422 if (Diff != 1 && Diff != 2 && Diff != 3)
10450 EVT VT =
N->getValueType(0);
10456 if (!Subtarget.hasShortForwardBranchOpt() ||
10466 bool SwapSelectOps;
10472 SwapSelectOps =
false;
10473 NonConstantVal = FalseVal;
10475 SwapSelectOps =
true;
10476 NonConstantVal = TrueVal;
10482 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
10530 EVT VT =
N->getValueType(0);
10545 if (!N0C->hasOneUse())
10547 int64_t C0 = N0C->getSExtValue();
10548 int64_t C1 = N1C->getSExtValue();
10550 if (C0 == -1 || C0 == 0 || C0 == 1 ||
isInt<12>(C1))
10557 }
else if ((C1 / C0 + 1) != 0 &&
isInt<12>(C1 / C0 + 1) &&
10561 }
else if ((C1 / C0 - 1) != 0 &&
isInt<12>(C1 / C0 - 1) &&
10580 EVT VT =
N->getValueType(0);
10616 EVT VT =
N->getValueType(0);
10626 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
10636 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
10665 EVT VT =
N->getValueType(0);
10708 EVT VT =
N->getValueType(0);
10728 EVT VT =
N->getValueType(0);
10735 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
10762 if (Subtarget.
is64Bit() && Subtarget.hasStdExtZbs() &&
10826 const APInt &Imm = ConstN00->getAPIntValue();
10827 if ((Imm + 1).isSignedIntN(12))
10845 if (
N.getOpcode() !=
ISD::SHL || !
N->hasOneUse())
10862 EVT SrcVT = Src.getValueType();
10866 NewElen = std::max(NewElen, 8U);
10888 EVT VT =
N->getValueType(0);
10891 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
10907 if (!isIntEqualitySetCC(
Cond))
10916 const APInt &C1 = N1C->getAPIntValue();
10934 EVT VT =
N->getValueType(0);
10940 Src.getOperand(0));
10948struct CombineResult;
10972struct NodeExtensionHelper {
10982 bool EnforceOneUse;
11003 return OrigOperand;
11016 std::optional<bool> SExt)
const {
11017 if (!SExt.has_value())
11018 return OrigOperand;
11020 MVT NarrowVT = getNarrowType(Root);
11023 if (
Source.getValueType() == NarrowVT)
11030 auto [
Mask, VL] = getMaskAndVL(Root);
11034 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
11051 static MVT getNarrowType(
const SDNode *Root) {
11056 assert(NarrowSize >= 8 &&
"Trying to extend something we can't represent");
11069 static unsigned getSameExtensionOpcode(
unsigned Opcode,
bool IsSExt) {
11088 static unsigned getSUOpcode(
unsigned Opcode) {
11095 static unsigned getWOpcode(
unsigned Opcode,
bool IsSExt) {
11106 using CombineToTry = std::function<std::optional<CombineResult>(
11107 SDNode * ,
const NodeExtensionHelper & ,
11108 const NodeExtensionHelper & )>;
11111 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
11116 SupportsZExt =
false;
11117 SupportsSExt =
false;
11118 EnforceOneUse =
true;
11122 SupportsZExt =
true;
11127 SupportsSExt =
true;
11134 EnforceOneUse =
false;
11151 unsigned ScalarBits =
Op.getValueSizeInBits();
11154 if (ScalarBits < EltBits)
11160 if (NarrowSize < 8)
11164 SupportsSExt =
true;
11167 SupportsZExt =
true;
11176 static bool isSupportedRoot(
const SDNode *Root) {
11193 assert(isSupportedRoot(Root) &&
"Trying to build an helper with an "
11194 "unsupported root");
11195 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
11206 if (OperandIdx == 1) {
11209 SupportsSExt = !SupportsZExt;
11210 std::tie(Mask, VL) = getMaskAndVL(Root);
11214 EnforceOneUse =
false;
11219 fillUpExtensionSupport(Root, DAG);
11225 bool isVLCompatible(
SDValue VL)
const {
11226 return this->VL !=
SDValue() && this->VL == VL;
11230 bool isMaskCompatible(
SDValue Mask)
const {
11231 return !CheckMask || (this->Mask !=
SDValue() && this->Mask ==
Mask);
11235 static std::pair<SDValue, SDValue> getMaskAndVL(
const SDNode *Root) {
11236 assert(isSupportedRoot(Root) &&
"Unexpected root");
11241 bool areVLAndMaskCompatible(
const SDNode *Root)
const {
11242 auto [
Mask, VL] = getMaskAndVL(Root);
11243 return isMaskCompatible(Mask) && isVLCompatible(VL);
11249 switch (
N->getOpcode()) {
11276struct CombineResult {
11278 unsigned TargetOpcode;
11281 std::optional<bool> SExtLHS;
11282 std::optional<bool> SExtRHS;
11286 NodeExtensionHelper
LHS;
11288 NodeExtensionHelper
RHS;
11290 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
11291 const NodeExtensionHelper &LHS, std::optional<bool> SExtLHS,
11292 const NodeExtensionHelper &RHS, std::optional<bool> SExtRHS)
11293 : TargetOpcode(TargetOpcode), SExtLHS(SExtLHS), SExtRHS(SExtRHS),
11294 Root(Root), LHS(LHS), RHS(RHS) {}
11301 std::tie(Mask, VL) = NodeExtensionHelper::getMaskAndVL(Root);
11304 LHS.getOrCreateExtendedOp(Root, DAG, SExtLHS),
11305 RHS.getOrCreateExtendedOp(Root, DAG, SExtRHS),
Merge,
11320static std::optional<CombineResult>
11321canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
11322 const NodeExtensionHelper &RHS,
bool AllowSExt,
11324 assert((AllowSExt || AllowZExt) &&
"Forgot to set what you want?");
11325 if (!LHS.areVLAndMaskCompatible(Root) || !RHS.areVLAndMaskCompatible(Root))
11326 return std::nullopt;
11327 if (AllowZExt && LHS.SupportsZExt && RHS.SupportsZExt)
11328 return CombineResult(NodeExtensionHelper::getSameExtensionOpcode(
11330 Root, LHS,
false, RHS,
11332 if (AllowSExt && LHS.SupportsSExt && RHS.SupportsSExt)
11333 return CombineResult(NodeExtensionHelper::getSameExtensionOpcode(
11335 Root, LHS,
true, RHS,
11337 return std::nullopt;
11346static std::optional<CombineResult>
11347canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
11348 const NodeExtensionHelper &RHS) {
11349 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS,
true,
11357static std::optional<CombineResult>
11358canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
11359 const NodeExtensionHelper &RHS) {
11360 if (!RHS.areVLAndMaskCompatible(Root))
11361 return std::nullopt;
11368 return CombineResult(
11369 NodeExtensionHelper::getWOpcode(Root->
getOpcode(),
false),
11370 Root, LHS, std::nullopt, RHS,
false);
11372 return CombineResult(
11373 NodeExtensionHelper::getWOpcode(Root->
getOpcode(),
true),
11374 Root, LHS, std::nullopt, RHS,
true);
11375 return std::nullopt;
11382static std::optional<CombineResult>
11383canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
11384 const NodeExtensionHelper &RHS) {
11385 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS,
true,
11393static std::optional<CombineResult>
11394canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
11395 const NodeExtensionHelper &RHS) {
11396 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS,
false,
11404static std::optional<CombineResult>
11405canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
11406 const NodeExtensionHelper &RHS) {
11407 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
11408 return std::nullopt;
11409 if (!LHS.areVLAndMaskCompatible(Root) || !RHS.areVLAndMaskCompatible(Root))
11410 return std::nullopt;
11411 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
11412 Root, LHS,
true, RHS,
false);
11416NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
11422 Strategies.
push_back(canFoldToVWWithSameExtension);
11428 Strategies.
push_back(canFoldToVWWithSameExtension);
11435 Strategies.
push_back(canFoldToVWWithSEXT);
11440 Strategies.
push_back(canFoldToVWWithZEXT);
11460 assert(NodeExtensionHelper::isSupportedRoot(
N) &&
11461 "Shouldn't have called this method");
11465 Inserted.insert(
N);
11468 while (!Worklist.
empty()) {
11470 if (!NodeExtensionHelper::isSupportedRoot(Root))
11473 NodeExtensionHelper
LHS(
N, 0, DAG);
11474 NodeExtensionHelper
RHS(
N, 1, DAG);
11475 auto AppendUsersIfNeeded = [&Worklist,
11476 &Inserted](
const NodeExtensionHelper &
Op) {
11477 if (
Op.needToPromoteOtherUsers()) {
11478 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
11479 if (Inserted.insert(TheUse).second)
11491 NodeExtensionHelper::getSupportedFoldings(
N);
11493 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
11494 bool Matched =
false;
11495 for (
int Attempt = 0;
11496 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
11499 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
11500 FoldingStrategies) {
11501 std::optional<CombineResult> Res = FoldingStrategy(
N, LHS, RHS);
11508 if (Res->SExtLHS.has_value())
11509 AppendUsersIfNeeded(LHS);
11510 if (Res->SExtRHS.has_value())
11511 AppendUsersIfNeeded(RHS);
11522 SDValue InputRootReplacement;
11529 for (CombineResult Res : CombinesToApply) {
11530 SDValue NewValue = Res.materialize(DAG);
11531 if (!InputRootReplacement) {
11533 "First element is expected to be the current node");
11534 InputRootReplacement = NewValue;
11539 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
11543 return InputRootReplacement;
11565 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
11570 if (LSNode1->
getOpcode() == ISD::LOAD) {
11573 if (MemVT == MVT::i32)
11579 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
11580 {LSNode1->getChain(), BasePtr,
11581 DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
11596 {LSNode1->getChain(), LSNode1->getOperand(1), LSNode2->getOperand(1),
11597 BasePtr, DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
11614 if (!Subtarget.hasVendorXTHeadMemPair())
11619 unsigned OpNum = LSNode1->
getOpcode() == ISD::LOAD ? 1 : 2;
11626 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
11629 return {
Ptr->getOperand(0), C1->getZExtValue()};
11633 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
11656 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
11659 if (Base1 != Base2)
11663 bool Valid =
false;
11664 if (MemVT == MVT::i32) {
11668 }
else if (MemVT == MVT::i64) {
11703 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
11711 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
11724 EVT VT =
N->getValueType(0);
11727 MVT SrcVT = Src.getSimpleValueType();
11728 MVT SrcContainerVT = SrcVT;
11730 SDValue XVal = Src.getOperand(0);
11757 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
11761 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
11774 if (VT != MVT::i32 && VT != XLenVT)
11803 EVT DstVT =
N->getValueType(0);
11804 if (DstVT != XLenVT)
11810 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
11818 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
11830 if (SatVT == DstVT)
11832 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
11838 Src = Src.getOperand(0);
11859 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
11865 EVT VT =
N->getValueType(0);
11925 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
11927 V.getOperand(2) == VL) {
11929 V = V.getOperand(0);
11936 bool NegA = invertIfNegative(
A);
11937 bool NegB = invertIfNegative(
B);
11938 bool NegC = invertIfNegative(
C);
11941 if (!NegA && !NegB && !NegC)
11945 if (
N->isTargetStrictFPOpcode())
11947 {N->getOperand(0), A, B, C, Mask, VL});
11957 if (
N->isTargetStrictFPOpcode())
11982 switch (
N->getOpcode()) {
12002 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
12008 assert(!
N->isTargetStrictFPOpcode() &&
"Unexpected opcode");
12036 Op1,
Merge, Mask, VL);
12053 (Op0 == Op1 && Op0IsExtend) ||
12058 Op0IsExtend =
false;
12060 Op1IsExtend =
false;
12063 if (!Op1IsExtend) {
12095 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
12100 uint64_t ShAmt =
N->getConstantOperandVal(1);
12208 if (!
Cond.hasOneUse())
12227 EVT VT =
Cond.getValueType();
12272 LHS.getOpcode() ==
ISD::SRA && RHSConst->isZero()) {
12273 LHS = LHS.getOperand(0);
12284 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
12292 RHS = LHS.getOperand(1);
12293 LHS = LHS.getOperand(0);
12302 RHS = LHS.getOperand(1);
12303 LHS = LHS.getOperand(0);
12314 uint64_t ShAmt = LHS.getConstantOperandVal(1);
12319 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
12360 bool Commutative =
true;
12361 switch (TrueVal.getOpcode()) {
12365 Commutative =
false;
12377 if (FalseVal == TrueVal.getOperand(0))
12379 else if (Commutative && FalseVal == TrueVal.getOperand(1))
12384 EVT VT =
N->getValueType(0);
12387 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
12392 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
12436 if (
Cond->getOperand(0) != CountZeroesArgument)
12452 CountZeroes, BitWidthMinusOne);
12461 if (Subtarget.hasShortForwardBranchOpt())
12464 SDValue TrueVal =
N->getOperand(1);
12465 SDValue FalseVal =
N->getOperand(2);
12479 EVT VT =
N->getValueType(0);
12491 !
SDValue(BaseLd, 0).hasOneUse())
12494 EVT BaseLdVT = BaseLd->getValueType(0);
12495 SDValue BasePtr = BaseLd->getBasePtr();
12504 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
12506 Ld->getValueType(0) != BaseLdVT)
12516 else if (
Offset != Stride)
12532 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(),
12547 unsigned WideScalarBitWidth =
12566 SDValue Ops[] = {BaseLd->getChain(),
12578 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
12584 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
12588 Ops, WideVecVT, MMO);
12609 auto IsVWMulOpc = [](
unsigned Opc) {
12636 if (AddMask != MulMask || AddVL != MulVL)
12641 "Unexpected opcode after VWMACC_VL");
12643 "Unexpected opcode after VWMACC_VL!");
12645 "Unexpected opcode after VWMUL_VL!");
12647 "Unexpected opcode after VWMUL_VL!");
12650 EVT VT =
N->getValueType(0);
12663 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
12674 switch (
N->getOpcode()) {
12696 APInt V =
C->getValueAPF().bitcastToAPInt();
12731 if (SimplifyDemandedLowBitsHelper(0, 32) ||
12732 SimplifyDemandedLowBitsHelper(1, 5))
12740 if (SimplifyDemandedLowBitsHelper(0, 32))
12757 MVT VT =
N->getSimpleValueType(0);
12766 "Unexpected value type!");
12814 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
12819 Src.getOperand(0));
12824 Src.getOperand(0), Src.getOperand(1));
12840 if (
N->getOperand(1).getOpcode() ==
ISD::XOR &&
12849 N->getOperand(0),
Cond);
12863 EVT VT =
N->getValueType(0);
12866 if (TrueV == FalseV)
12898 {LHS, RHS, CC, TrueV, FalseV});
12900 if (!Subtarget.hasShortForwardBranchOpt()) {
12952 N->getOperand(0), LHS, RHS,
CC,
N->getOperand(4));
12965 EVT VT =
N->getValueType(0);
12977 if (In2.
getOpcode() != ISD::FP_EXTEND &&
12986 DAG.
getNode(ISD::FNEG,
DL, VT, NewFPExtRound));
12989 case ISD::MSCATTER:
12990 case ISD::VP_GATHER:
12991 case ISD::VP_SCATTER: {
12995 bool IsIndexSigned =
false;
12997 Index = VPGSN->getIndex();
12998 ScaleOp = VPGSN->getScale();
12999 IsIndexSigned = VPGSN->isIndexSigned();
13000 assert(!VPGSN->isIndexScaled() &&
13001 "Scaled gather/scatter should not be formed");
13004 Index = MGSN->getIndex();
13005 ScaleOp = MGSN->getScale();
13006 IsIndexSigned = MGSN->isIndexSigned();
13007 assert(!MGSN->isIndexScaled() &&
13008 "Scaled gather/scatter should not be formed");
13011 EVT IndexVT =
Index.getValueType();
13015 bool NeedsIdxLegalization =
13017 if (!NeedsIdxLegalization)
13035 {VPGN->getChain(), VPGN->getBasePtr(), Index,
13036 ScaleOp, VPGN->getMask(),
13037 VPGN->getVectorLength()},
13038 VPGN->getMemOperand(), NewIndexTy);
13041 {VPSN->getChain(), VPSN->getValue(),
13042 VPSN->getBasePtr(), Index, ScaleOp,
13043 VPSN->getMask(), VPSN->getVectorLength()},
13044 VPSN->getMemOperand(), NewIndexTy);
13047 N->getVTList(), MGN->getMemoryVT(),
DL,
13048 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
13049 MGN->getBasePtr(), Index, ScaleOp},
13050 MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
13053 N->getVTList(), MSN->getMemoryVT(),
DL,
13054 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
13056 MSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
13066 EVT VT =
N->getValueType(0);
13069 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
13084 EVT VT =
N->getValueType(0);
13088 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
13123 if (
N->getOpcode() != ISD::STORE)
13127 SDValue Chain = Store->getChain();
13128 EVT MemVT = Store->getMemoryVT();
13129 SDValue Val = Store->getValue();
13132 bool IsScalarizable =
13134 Store->isSimple() &&
13161 Subtarget.getFeatureBits(),
true) <= 2 &&
13163 NewVT, *Store->getMemOperand())) {
13165 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
13166 Store->getPointerInfo(), Store->getOriginalAlign(),
13167 Store->getMemOperand()->getFlags());
13177 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
13179 L->getMemoryVT() == MemVT) {
13182 NewVT, *Store->getMemOperand()) &&
13184 NewVT, *L->getMemOperand())) {
13186 L->getPointerInfo(), L->getOriginalAlign(),
13187 L->getMemOperand()->getFlags());
13188 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
13189 Store->getPointerInfo(), Store->getOriginalAlign(),
13190 Store->getMemOperand()->getFlags());
13202 MVT VecVT = Src.getSimpleValueType();
13209 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
13212 Store->getMemOperand(), Store->getAddressingMode(),
13213 Store->isTruncatingStore(),
false);
13220 EVT VT =
N->getValueType(0);
13236 unsigned ScalarSize =
N->getOperand(1).getValueSizeInBits();
13237 unsigned EltWidth =
N->getValueType(0).getScalarSizeInBits();
13238 if (ScalarSize > EltWidth &&
N->getOperand(0).isUndef())
13239 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
13249 if (
N->getOperand(0).isUndef() &&
13252 Src.getOperand(0).getValueType().isScalableVector()) {
13253 EVT VT =
N->getValueType(0);
13254 EVT SrcVT = Src.getOperand(0).getValueType();
13258 return Src.getOperand(0);
13267 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
13272 case Intrinsic::riscv_vcpop:
13273 case Intrinsic::riscv_vcpop_mask:
13274 case Intrinsic::riscv_vfirst:
13275 case Intrinsic::riscv_vfirst_mask: {
13277 if (IntNo == Intrinsic::riscv_vcpop_mask ||
13278 IntNo == Intrinsic::riscv_vfirst_mask)
13284 EVT VT =
N->getValueType(0);
13285 if (IntNo == Intrinsic::riscv_vfirst ||
13286 IntNo == Intrinsic::riscv_vfirst_mask)
13290 case Intrinsic::riscv_vloxei:
13291 case Intrinsic::riscv_vloxei_mask:
13292 case Intrinsic::riscv_vluxei:
13293 case Intrinsic::riscv_vluxei_mask:
13294 case Intrinsic::riscv_vsoxei:
13295 case Intrinsic::riscv_vsoxei_mask:
13296 case Intrinsic::riscv_vsuxei:
13297 case Intrinsic::riscv_vsuxei_mask:
13303 Ops, MemSD->getMemoryVT(),
13304 MemSD->getMemOperand());
13309 case ISD::BITCAST: {
13312 EVT VT =
N->getValueType(0);
13316 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
13335 EVT XVT,
unsigned KeptBits)
const {
13340 if (XVT != MVT::i32 && XVT != MVT::i64)
13344 if (KeptBits == 32 || KeptBits == 64)
13348 return Subtarget.hasStdExtZbb() &&
13349 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
13357 "Expected shift op");
13366 if (Ty.isScalarInteger() &&
13371 const APInt &C1Int = C1->getAPIntValue();
13372 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
13390 Subtarget.getFeatureBits(),
13393 ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
13398 if (C1Cost < ShiftedC1Cost)
13412 EVT VT =
Op.getValueType();
13416 unsigned Opcode =
Op.getOpcode();
13424 const APInt &Mask =
C->getAPIntValue();
13433 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
13434 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
13436 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
13437 if (NewMask == Mask)
13442 Op.getOperand(0), NewC);
13455 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
13456 if (IsLegalMask(NewMask))
13457 return UseMask(NewMask);
13460 if (VT == MVT::i64) {
13462 if (IsLegalMask(NewMask))
13463 return UseMask(NewMask);
13478 APInt NewMask = ShrunkMask;
13479 if (MinSignedBits <= 12)
13481 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
13487 assert(IsLegalMask(NewMask));
13488 return UseMask(NewMask);
13492 static const uint64_t GREVMasks[] = {
13493 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
13494 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
13496 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
13497 unsigned Shift = 1 << Stage;
13498 if (ShAmt & Shift) {
13500 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
13512 const APInt &DemandedElts,
13514 unsigned Depth)
const {
13516 unsigned Opc =
Op.getOpcode();
13521 "Should use MaskedValueIsZero if you don't know whether Op"
13522 " is a target node!");
13596 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
13599 if (MaxVLenB == MinVLenB)
13616 case Intrinsic::riscv_vsetvli:
13617 case Intrinsic::riscv_vsetvlimax:
13631 unsigned Depth)
const {
13632 switch (
Op.getOpcode()) {
13638 if (Tmp == 1)
return 1;
13641 return std::min(Tmp, Tmp2);
13653 if (Tmp < 33)
return 1;
13678 unsigned XLen = Subtarget.
getXLen();
13679 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
13680 if (EltBits <= XLen)
13681 return XLen - EltBits + 1;
13685 unsigned IntNo =
Op.getConstantOperandVal(1);
13689 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
13690 case Intrinsic::riscv_masked_atomicrmw_add_i64:
13691 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
13692 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
13693 case Intrinsic::riscv_masked_atomicrmw_max_i64:
13694 case Intrinsic::riscv_masked_atomicrmw_min_i64:
13695 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
13696 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
13697 case Intrinsic::riscv_masked_cmpxchg_i64:
13705 assert(Subtarget.hasStdExtA());
13716 assert(Ld &&
"Unexpected null LoadSDNode");
13725 if (!CNode || CNode->isMachineConstantPoolEntry() ||
13726 CNode->getOffset() != 0)
13734 auto *CNode = GetSupportedConstantPool(
Ptr);
13735 if (!CNode || CNode->getTargetFlags() != 0)
13738 return CNode->getConstVal();
13746 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
13747 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
13753 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
13756 return CNodeLo->getConstVal();
13761 assert(
MI.getOpcode() == RISCV::ReadCycleWide &&
"Unexpected instruction");
13792 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
13799 .
addImm(RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding)
13802 .
addImm(RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding)
13804 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
13805 .
addImm(RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding)
13816 MI.eraseFromParent();
13824 assert((
MI.getOpcode() == RISCV::SplitF64Pseudo ||
13825 MI.getOpcode() == RISCV::SplitF64Pseudo_INX) &&
13826 "Unexpected instruction");
13834 Register SrcReg =
MI.getOperand(2).getReg();
13837 ? &RISCV::GPRPF64RegClass
13838 : &RISCV::FPR64RegClass;
13856 MI.eraseFromParent();
13863 assert((
MI.getOpcode() == RISCV::BuildPairF64Pseudo ||
13864 MI.getOpcode() == RISCV::BuildPairF64Pseudo_INX) &&
13865 "Unexpected instruction");
13871 Register DstReg =
MI.getOperand(0).getReg();
13876 MI.getOpcode() == RISCV::BuildPairF64Pseudo_INX ? &RISCV::GPRPF64RegClass
13877 : &RISCV::FPR64RegClass;
13896 MI.eraseFromParent();
13901 switch (
MI.getOpcode()) {
13904 case RISCV::Select_GPR_Using_CC_GPR:
13905 case RISCV::Select_FPR16_Using_CC_GPR:
13906 case RISCV::Select_FPR16INX_Using_CC_GPR:
13907 case RISCV::Select_FPR32_Using_CC_GPR:
13908 case RISCV::Select_FPR32INX_Using_CC_GPR:
13909 case RISCV::Select_FPR64_Using_CC_GPR:
13910 case RISCV::Select_FPR64INX_Using_CC_GPR:
13911 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
13917 unsigned RelOpcode,
unsigned EqOpcode,
13920 Register DstReg =
MI.getOperand(0).getReg();
13921 Register Src1Reg =
MI.getOperand(1).getReg();
13922 Register Src2Reg =
MI.getOperand(2).getReg();
13924 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
13948 MI.eraseFromParent();
13999 F->insert(It, FirstMBB);
14000 F->insert(It, SecondMBB);
14001 F->insert(It, SinkMBB);
14050 First.eraseFromParent();
14093 SelectDests.
insert(
MI.getOperand(0).getReg());
14097 if (
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
Next != BB->
end() &&
14098 Next->getOpcode() ==
MI.getOpcode() &&
14099 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
14100 Next->getOperand(5).isKill()) {
14105 SequenceMBBI !=
E; ++SequenceMBBI) {
14106 if (SequenceMBBI->isDebugInstr())
14109 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
14110 SequenceMBBI->getOperand(2).getReg() != RHS ||
14111 SequenceMBBI->getOperand(3).getImm() !=
CC ||
14112 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
14113 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
14115 LastSelectPseudo = &*SequenceMBBI;
14117 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
14120 if (SequenceMBBI->hasUnmodeledSideEffects() ||
14121 SequenceMBBI->mayLoadOrStore() ||
14122 SequenceMBBI->usesCustomInsertionHook())
14125 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
14141 F->insert(
I, TailMBB);
14145 TailMBB->
push_back(DebugInstr->removeFromParent());
14149 TailMBB->
splice(TailMBB->
end(), HeadMBB,
14168 auto SelectMBBI =
MI.getIterator();
14169 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
14170 auto InsertionPoint = TailMBB->
begin();
14171 while (SelectMBBI != SelectEnd) {
14172 auto Next = std::next(SelectMBBI);
14175 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
14176 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
14177 .
addReg(SelectMBBI->getOperand(4).getReg())
14179 .
addReg(SelectMBBI->getOperand(5).getReg())
14197 Register SavedFRM =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
14199 assert(
MI.getNumOperands() == 8 ||
MI.getNumOperands() == 7);
14200 unsigned FRMIdx =
MI.getNumOperands() == 8 ? 4 : 3;
14204 .
addImm(
MI.getOperand(FRMIdx).getImm());
14209 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++)
14211 MIB = MIB.add(
MI.getOperand(
I));
14227 MI.eraseFromParent();
14234 unsigned CVTFOpc) {
14240 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
14253 .
add(
MI.getOperand(1))
14254 .
add(
MI.getOperand(2))
14255 .
add(
MI.getOperand(3))
14257 .
add(
MI.getOperand(4))
14258 .
add(
MI.getOperand(5))
14259 .
add(
MI.getOperand(6))
14266 .
add(
MI.getOperand(0))
14267 .
add(
MI.getOperand(1))
14269 .
add(
MI.getOperand(3))
14271 .
add(
MI.getOperand(4))
14272 .
add(
MI.getOperand(5))
14273 .
add(
MI.getOperand(6))
14283 MI.eraseFromParent();
14289 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
14291 switch (
MI.getOpcode()) {
14294 case RISCV::PseudoFROUND_H:
14295 CmpOpc = RISCV::FLT_H;
14296 F2IOpc = RISCV::FCVT_W_H;
14297 I2FOpc = RISCV::FCVT_H_W;
14298 FSGNJOpc = RISCV::FSGNJ_H;
14299 FSGNJXOpc = RISCV::FSGNJX_H;
14300 RC = &RISCV::FPR16RegClass;
14302 case RISCV::PseudoFROUND_H_INX:
14303 CmpOpc = RISCV::FLT_H_INX;
14304 F2IOpc = RISCV::FCVT_W_H_INX;
14305 I2FOpc = RISCV::FCVT_H_W_INX;
14306 FSGNJOpc = RISCV::FSGNJ_H_INX;
14307 FSGNJXOpc = RISCV::FSGNJX_H_INX;
14308 RC = &RISCV::GPRF16RegClass;
14310 case RISCV::PseudoFROUND_S:
14311 CmpOpc = RISCV::FLT_S;
14312 F2IOpc = RISCV::FCVT_W_S;
14313 I2FOpc = RISCV::FCVT_S_W;
14314 FSGNJOpc = RISCV::FSGNJ_S;
14315 FSGNJXOpc = RISCV::FSGNJX_S;
14316 RC = &RISCV::FPR32RegClass;
14318 case RISCV::PseudoFROUND_S_INX:
14319 CmpOpc = RISCV::FLT_S_INX;
14320 F2IOpc = RISCV::FCVT_W_S_INX;
14321 I2FOpc = RISCV::FCVT_S_W_INX;
14322 FSGNJOpc = RISCV::FSGNJ_S_INX;
14323 FSGNJXOpc = RISCV::FSGNJX_S_INX;
14324 RC = &RISCV::GPRF32RegClass;
14326 case RISCV::PseudoFROUND_D:
14328 CmpOpc = RISCV::FLT_D;
14329 F2IOpc = RISCV::FCVT_L_D;
14330 I2FOpc = RISCV::FCVT_D_L;
14331 FSGNJOpc = RISCV::FSGNJ_D;
14332 FSGNJXOpc = RISCV::FSGNJX_D;
14333 RC = &RISCV::FPR64RegClass;
14335 case RISCV::PseudoFROUND_D_INX:
14337 CmpOpc = RISCV::FLT_D_INX;
14338 F2IOpc = RISCV::FCVT_L_D_INX;
14339 I2FOpc = RISCV::FCVT_D_L_INX;
14340 FSGNJOpc = RISCV::FSGNJ_D_INX;
14341 FSGNJXOpc = RISCV::FSGNJX_D_INX;
14342 RC = &RISCV::GPRRegClass;
14355 F->insert(
I, DoneMBB);
14366 Register DstReg =
MI.getOperand(0).getReg();
14367 Register SrcReg =
MI.getOperand(1).getReg();
14368 Register MaxReg =
MI.getOperand(2).getReg();
14369 int64_t FRM =
MI.getOperand(3).getImm();
14374 Register FabsReg =
MRI.createVirtualRegister(RC);
14378 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
14393 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
14415 MI.eraseFromParent();
14422 switch (
MI.getOpcode()) {
14425 case RISCV::ReadCycleWide:
14427 "ReadCycleWrite is only to be used on riscv32");
14429 case RISCV::Select_GPR_Using_CC_GPR:
14430 case RISCV::Select_FPR16_Using_CC_GPR:
14431 case RISCV::Select_FPR16INX_Using_CC_GPR:
14432 case RISCV::Select_FPR32_Using_CC_GPR:
14433 case RISCV::Select_FPR32INX_Using_CC_GPR:
14434 case RISCV::Select_FPR64_Using_CC_GPR:
14435 case RISCV::Select_FPR64INX_Using_CC_GPR:
14436 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
14438 case RISCV::BuildPairF64Pseudo:
14439 case RISCV::BuildPairF64Pseudo_INX:
14441 case RISCV::SplitF64Pseudo:
14442 case RISCV::SplitF64Pseudo_INX:
14444 case RISCV::PseudoQuietFLE_H:
14446 case RISCV::PseudoQuietFLE_H_INX:
14447 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
14448 case RISCV::PseudoQuietFLT_H:
14450 case RISCV::PseudoQuietFLT_H_INX:
14451 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
14452 case RISCV::PseudoQuietFLE_S:
14454 case RISCV::PseudoQuietFLE_S_INX:
14455 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
14456 case RISCV::PseudoQuietFLT_S:
14458 case RISCV::PseudoQuietFLT_S_INX:
14459 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
14460 case RISCV::PseudoQuietFLE_D:
14462 case RISCV::PseudoQuietFLE_D_INX:
14463 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
14464 case RISCV::PseudoQuietFLE_D_IN32X:
14467 case RISCV::PseudoQuietFLT_D:
14469 case RISCV::PseudoQuietFLT_D_INX:
14470 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
14471 case RISCV::PseudoQuietFLT_D_IN32X:
14475#define PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, LMUL) \
14476 case RISCV::RMOpc##_##LMUL: \
14477 return emitVFCVT_RM(MI, BB, RISCV::Opc##_##LMUL); \
14478 case RISCV::RMOpc##_##LMUL##_MASK: \
14479 return emitVFCVT_RM(MI, BB, RISCV::Opc##_##LMUL##_MASK);
14481#define PseudoVFCVT_RM_CASE(RMOpc, Opc) \
14482 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, M1) \
14483 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, M2) \
14484 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, M4) \
14485 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, MF2) \
14486 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, MF4)
14488#define PseudoVFCVT_RM_CASE_M8(RMOpc, Opc) \
14489 PseudoVFCVT_RM_CASE(RMOpc, Opc) \
14490 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, M8)
14492#define PseudoVFCVT_RM_CASE_MF8(RMOpc, Opc) \
14493 PseudoVFCVT_RM_CASE(RMOpc, Opc) \
14494 PseudoVFCVT_RM_LMUL_CASE(RMOpc, Opc, MF8)
14512 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
14514 RISCV::PseudoVFCVT_F_X_V_M1_MASK);
14515 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
14517 RISCV::PseudoVFCVT_F_X_V_M2_MASK);
14518 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
14520 RISCV::PseudoVFCVT_F_X_V_M4_MASK);
14521 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
14523 RISCV::PseudoVFCVT_F_X_V_M8_MASK);
14524 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
14526 RISCV::PseudoVFCVT_F_X_V_MF2_MASK);
14527 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
14529 RISCV::PseudoVFCVT_F_X_V_MF4_MASK);
14530 case RISCV::PseudoFROUND_H:
14531 case RISCV::PseudoFROUND_H_INX:
14532 case RISCV::PseudoFROUND_S:
14533 case RISCV::PseudoFROUND_S_INX:
14534 case RISCV::PseudoFROUND_D:
14535 case RISCV::PseudoFROUND_D_INX:
14536 case RISCV::PseudoFROUND_D_IN32X:
14546 return std::nullopt;
14561 unsigned FRMImm =
MI.getOperand(*RoundModeIdx).getImm();
14569 unsigned Opc =
MI.getOpcode();
14576 if (
MI.readsRegister(RISCV::FRM))
14608 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
14609 RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
14612 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
14613 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
14616 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
14617 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
14620 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
14621 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
14625 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
14626 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
14627 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
14629 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
14630 RISCV::V20M2, RISCV::V22M2};
14641 unsigned XLenInBytes = XLen / 8;
14652 State.AllocateStack(XLenInBytes, StackAlign),
14655 ValNo2, ValVT2, State.AllocateStack(XLenInBytes,
Align(XLenInBytes)),
14667 ValNo2, ValVT2, State.AllocateStack(XLenInBytes,
Align(XLenInBytes)),
14675 std::optional<unsigned> FirstMaskArgument,
14678 if (RC == &RISCV::VRRegClass) {
14682 if (FirstMaskArgument && ValNo == *FirstMaskArgument)
14683 return State.AllocateReg(RISCV::V0);
14684 return State.AllocateReg(
ArgVRs);
14686 if (RC == &RISCV::VRM2RegClass)
14687 return State.AllocateReg(
ArgVRM2s);
14688 if (RC == &RISCV::VRM4RegClass)
14689 return State.AllocateReg(
ArgVRM4s);
14690 if (RC == &RISCV::VRM8RegClass)
14691 return State.AllocateReg(
ArgVRM8s);
14700 std::optional<unsigned> FirstMaskArgument) {
14701 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
14702 assert(XLen == 32 || XLen == 64);
14703 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
14707 if (ArgFlags.
isNest()) {
14708 if (
unsigned Reg = State.AllocateReg(RISCV::X7)) {
14716 if (!LocVT.
isVector() && IsRet && ValNo > 1)
14721 bool UseGPRForF16_F32 =
true;
14724 bool UseGPRForF64 =
true;
14734 UseGPRForF16_F32 = !IsFixed;
14738 UseGPRForF16_F32 = !IsFixed;
14739 UseGPRForF64 = !IsFixed;
14745 UseGPRForF16_F32 =
true;
14746 UseGPRForF64 =
true;
14753 if (UseGPRForF16_F32 &&
14754 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
14757 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
14769 unsigned TwoXLenInBytes = (2 * XLen) / 8;
14771 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
14772 unsigned RegIdx = State.getFirstUnallocated(
ArgGPRs);
14774 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
14780 State.getPendingArgFlags();
14783 "PendingLocs and PendingArgFlags out of sync");
14787 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
14789 "Can't lower f64 if it is split");
14802 if (!State.AllocateReg(
ArgGPRs))
14803 State.AllocateStack(4,
Align(4));
14830 PendingLocs.
size() <= 2) {
14831 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
14836 PendingLocs.
clear();
14837 PendingArgFlags.
clear();
14844 unsigned StoreSizeBytes = XLen / 8;
14847 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
14849 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
14851 else if (ValVT == MVT::f64 && !UseGPRForF64)
14854 Reg =
allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
14863 if ((Reg = State.AllocateReg(
ArgGPRs))) {
14879 Reg = State.AllocateReg(
ArgGPRs);
14883 Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
14887 if (!PendingLocs.
empty()) {
14889 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
14891 for (
auto &It : PendingLocs) {
14893 It.convertToReg(Reg);
14898 PendingLocs.clear();
14899 PendingArgFlags.
clear();
14903 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
14905 "Expected an XLenVT or vector types at this stage");
14923template <
typename ArgTy>
14925 for (
const auto &ArgIdx :
enumerate(Args)) {
14926 MVT ArgVT = ArgIdx.value().VT;
14928 return ArgIdx.index();
14930 return std::nullopt;
14933void RISCVTargetLowering::analyzeInputArgs(
14936 RISCVCCAssignFn Fn)
const {
14937 unsigned NumArgs =
Ins.size();
14940 std::optional<unsigned> FirstMaskArgument;
14944 for (
unsigned i = 0; i != NumArgs; ++i) {
14948 Type *ArgTy =
nullptr;
14950 ArgTy = FType->getReturnType();
14951 else if (Ins[i].isOrigArg())
14952 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
14956 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
14957 FirstMaskArgument)) {
14958 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
14965void RISCVTargetLowering::analyzeOutputArgs(
14968 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
14969 unsigned NumArgs = Outs.
size();
14971 std::optional<unsigned> FirstMaskArgument;
14975 for (
unsigned i = 0; i != NumArgs; i++) {
14976 MVT ArgVT = Outs[i].VT;
14978 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
14982 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
14983 FirstMaskArgument)) {
14984 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
15032 if (In.isOrigArg()) {
15037 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
15038 (
BitWidth < 32 && In.Flags.isZExt())) {
15070 Val = DAG.
getNode(ISD::BITCAST,
DL, LocVT, Val);
15107 ExtType,
DL, LocVT, Chain, FIN,
15125 return DAG.
getLoad(MVT::f64,
DL, Chain, FIN,
15131 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
15143 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
15153 unsigned ValNo,
MVT ValVT,
MVT LocVT,
15156 bool IsFixed,
bool IsRet,
Type *OrigTy,
15158 std::optional<unsigned> FirstMaskArgument) {
15162 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
15163 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
15164 RISCV::X29, RISCV::X30, RISCV::X31};
15166 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
15167 if (
unsigned Reg = State.AllocateReg(GPRList)) {
15175 if (LocVT == MVT::f16 &&
15176 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
15178 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
15179 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
15180 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
15181 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
15182 if (
unsigned Reg = State.AllocateReg(FPR16List)) {
15188 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
15190 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
15191 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
15192 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
15193 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
15194 if (
unsigned Reg = State.AllocateReg(FPR32List)) {
15200 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
15202 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
15203 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
15204 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
15205 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
15206 if (
unsigned Reg = State.AllocateReg(FPR64List)) {
15213 if ((LocVT == MVT::f16 &&
15214 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
15215 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
15216 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
15217 Subtarget.hasStdExtZdinx())) {
15218 if (
unsigned Reg = State.AllocateReg(GPRList)) {
15224 if (LocVT == MVT::f16) {
15225 unsigned Offset2 = State.AllocateStack(2,
Align(2));
15230 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
15231 unsigned Offset4 = State.AllocateStack(4,
Align(4));
15236 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
15237 unsigned Offset5 = State.AllocateStack(8,
Align(8));
15252 if (
unsigned GPRReg = State.AllocateReg(GPRList)) {
15260 State.AllocateStack(ValVT.
getStoreSize(), StackAlign);
15278 if (ArgFlags.
isNest()) {
15280 "Attribute 'nest' is not supported in GHC calling convention");
15284 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
15285 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
15287 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
15290 if (
unsigned Reg = State.AllocateReg(GPRList)) {
15299 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
15302 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
15303 RISCV::F18_F, RISCV::F19_F,
15304 RISCV::F20_F, RISCV::F21_F};
15305 if (
unsigned Reg = State.AllocateReg(FPR32List)) {
15311 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
15314 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
15315 RISCV::F24_D, RISCV::F25_D,
15316 RISCV::F26_D, RISCV::F27_D};
15317 if (
unsigned Reg = State.AllocateReg(FPR64List)) {
15323 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
15324 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
15326 if (
unsigned Reg = State.AllocateReg(GPRList)) {
15338 SDValue Chain, CallingConv::ID CallConv,
bool IsVarArg,
15344 switch (CallConv) {
15353 "(Zdinx/D) instruction set extensions");
15357 if (Func.hasFnAttribute(
"interrupt")) {
15358 if (!Func.arg_empty())
15360 "Functions with the interrupt attribute cannot have arguments!");
15365 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
15367 "Function interrupt attribute argument not supported!");
15372 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
15374 std::vector<SDValue> OutChains;
15383 analyzeInputArgs(MF, CCInfo, Ins,
false,
15387 for (
unsigned i = 0, e = ArgLocs.
size(); i != e; ++i) {
15407 unsigned ArgIndex = Ins[i].OrigArgIndex;
15408 unsigned ArgPartOffset = Ins[i].PartOffset;
15410 while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
15412 unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
15441 int VaArgOffset, VarArgsSaveSize;
15445 if (ArgRegs.
size() == Idx) {
15447 VarArgsSaveSize = 0;
15449 VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() - Idx);
15450 VaArgOffset = -VarArgsSaveSize;
15463 VarArgsSaveSize += XLenInBytes;
15468 for (
unsigned I = Idx;
I < ArgRegs.
size();
15469 ++
I, VaArgOffset += XLenInBytes) {
15470 const Register Reg = RegInfo.createVirtualRegister(RC);
15471 RegInfo.addLiveIn(ArgRegs[
I], Reg);
15479 ->setValue((
Value *)
nullptr);
15480 OutChains.push_back(Store);
15487 if (!OutChains.empty()) {
15488 OutChains.push_back(Chain);
15498bool RISCVTargetLowering::isEligibleForTailCallOptimization(
15502 auto CalleeCC = CLI.CallConv;
15503 auto &Outs = CLI.Outs;
15505 auto CallerCC = Caller.getCallingConv();
15512 if (Caller.hasFnAttribute(
"interrupt"))
15527 for (
auto &VA : ArgLocs)
15533 auto IsCallerStructRet = Caller.hasStructRetAttr();
15534 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
15535 if (IsCallerStructRet || IsCalleeStructRet)
15540 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
15541 if (CalleeCC != CallerCC) {
15542 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
15543 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
15550 for (
auto &
Arg : Outs)
15551 if (
Arg.Flags.isByVal())
15574 CallingConv::ID CallConv = CLI.
CallConv;
15588 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
15594 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
15600 "site marked musttail");
15607 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
15609 if (!Flags.isByVal())
15613 unsigned Size = Flags.getByValSize();
15614 Align Alignment = Flags.getNonZeroByValAlign();
15621 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr,
Arg, SizeNode, Alignment,
15635 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(); i != e; ++i) {
15637 SDValue ArgValue = OutVals[i];
15641 bool IsF64OnRV32DSoftABI =
15643 if (IsF64OnRV32DSoftABI && VA.
isRegLoc()) {
15652 if (RegLo == RISCV::X17) {
15655 if (!StackPtr.getNode())
15662 assert(RegLo < RISCV::X31 &&
"Invalid register pair");
15664 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
15685 unsigned ArgIndex = Outs[i].OrigArgIndex;
15686 unsigned ArgPartOffset = Outs[i].PartOffset;
15692 while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
15693 SDValue PartValue = OutVals[i + 1];
15694 unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
15707 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
15709 for (
const auto &Part : Parts) {
15710 SDValue PartValue = Part.first;
15711 SDValue PartOffset = Part.second;
15718 ArgValue = SpillSlot;
15724 if (Flags.isByVal())
15725 ArgValue = ByValArgs[j++];
15732 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
15733 "for passing parameters");
15736 if (!StackPtr.getNode())
15749 if (!MemOpChains.
empty())
15755 for (
auto &Reg : RegsToPass) {
15756 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
15763 validateCCReservedRegs(RegsToPass, MF);
15768 "Return address register required, but has been reserved."});
15798 for (
auto &Reg : RegsToPass)
15804 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
15805 assert(Mask &&
"Missing call preserved mask for calling convention");
15814 "Unexpected CFI type for a direct call");
15844 for (
auto &VA : RVLocs) {
15852 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
15853 assert(VA.getLocReg() ==
ArgGPRs[0] &&
"Unexpected reg assignment");
15874 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
15876 std::optional<unsigned> FirstMaskArgument;
15880 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
15881 MVT VT = Outs[i].VT;
15885 ArgFlags, CCInfo,
true,
true,
nullptr,
15886 *
this, FirstMaskArgument))
15918 for (
unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
15927 DAG.
getVTList(MVT::i32, MVT::i32), Val);
15931 assert(RegLo < RISCV::X31 &&
"Invalid register pair");
15938 "Return value register required, but has been reserved."});
15954 "Return value register required, but has been reserved."});
15976 if (Func.hasFnAttribute(
"interrupt")) {
15977 if (!Func.getReturnType()->isVoidTy())
15979 "Functions with the interrupt attribute must have void return type!");
15985 if (Kind ==
"supervisor")
15991 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
15994void RISCVTargetLowering::validateCCReservedRegs(
15995 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
16004 F,
"Argument register required, but has been reserved."});
16010 if (
N->getNumValues() != 1)
16012 if (!
N->hasNUsesOfValue(1, 0))
16015 SDNode *Copy = *
N->use_begin();
16017 if (Copy->getOpcode() == ISD::BITCAST) {
16029 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
16033 bool HasRet =
false;
16034 for (
SDNode *Node : Copy->uses()) {
16042 Chain = Copy->getOperand(0);
16051#define NODE_NAME_CASE(NODE) \
16052 case RISCVISD::NODE: \
16053 return "RISCVISD::" #NODE;
16276#undef NODE_NAME_CASE
16283 if (Constraint.
size() == 1) {
16284 switch (Constraint[0]) {
16299 if (Constraint ==
"vr" || Constraint ==
"vm")
16305std::pair<unsigned, const TargetRegisterClass *>
16311 if (Constraint.
size() == 1) {
16312 switch (Constraint[0]) {
16317 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
16320 return std::make_pair(0U, &RISCV::FPR16RegClass);
16321 if (Subtarget.hasStdExtF() && VT == MVT::f32)
16322 return std::make_pair(0U, &RISCV::FPR32RegClass);
16323 if (Subtarget.hasStdExtD() && VT == MVT::f64)
16324 return std::make_pair(0U, &RISCV::FPR64RegClass);
16329 }
else if (Constraint ==
"vr") {
16330 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
16331 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
16333 return std::make_pair(0U, RC);
16335 }
else if (Constraint ==
"vm") {
16336 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
16337 return std::make_pair(0U, &RISCV::VMV0RegClass);
16345 .
Case(
"{zero}", RISCV::X0)
16346 .
Case(
"{ra}", RISCV::X1)
16347 .
Case(
"{sp}", RISCV::X2)
16348 .
Case(
"{gp}", RISCV::X3)
16349 .
Case(
"{tp}", RISCV::X4)
16350 .
Case(
"{t0}", RISCV::X5)
16351 .
Case(
"{t1}", RISCV::X6)
16352 .
Case(
"{t2}", RISCV::X7)
16353 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
16354 .
Case(
"{s1}", RISCV::X9)
16355 .
Case(
"{a0}", RISCV::X10)
16356 .
Case(
"{a1}", RISCV::X11)
16357 .
Case(
"{a2}", RISCV::X12)
16358 .
Case(
"{a3}", RISCV::X13)
16359 .
Case(
"{a4}", RISCV::X14)
16360 .
Case(
"{a5}", RISCV::X15)
16361 .
Case(
"{a6}", RISCV::X16)
16362 .
Case(
"{a7}", RISCV::X17)
16363 .
Case(
"{s2}", RISCV::X18)
16364 .
Case(
"{s3}", RISCV::X19)
16365 .
Case(
"{s4}", RISCV::X20)
16366 .
Case(
"{s5}", RISCV::X21)
16367 .
Case(
"{s6}", RISCV::X22)
16368 .
Case(
"{s7}", RISCV::X23)
16369 .
Case(
"{s8}", RISCV::X24)
16370 .
Case(
"{s9}", RISCV::X25)
16371 .
Case(
"{s10}", RISCV::X26)
16372 .
Case(
"{s11}", RISCV::X27)
16373 .
Case(
"{t3}", RISCV::X28)
16374 .
Case(
"{t4}", RISCV::X29)
16375 .
Case(
"{t5}", RISCV::X30)
16376 .
Case(
"{t6}", RISCV::X31)
16378 if (XRegFromAlias != RISCV::NoRegister)
16379 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
16388 if (Subtarget.hasStdExtF()) {
16390 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
16391 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
16392 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
16393 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
16394 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
16395 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
16396 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
16397 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
16398 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
16399 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
16400 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
16401 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
16402 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
16403 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
16404 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
16405 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
16406 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
16407 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
16408 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
16409 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
16410 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
16411 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
16412 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
16413 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
16414 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
16415 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
16416 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
16417 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
16418 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
16419 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
16420 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
16421 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
16423 if (FReg != RISCV::NoRegister) {
16424 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
16425 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
16426 unsigned RegNo = FReg - RISCV::F0_F;
16427 unsigned DReg = RISCV::F0_D + RegNo;
16428 return std::make_pair(DReg, &RISCV::FPR64RegClass);
16430 if (VT == MVT::f32 || VT == MVT::Other)
16431 return std::make_pair(FReg, &RISCV::FPR32RegClass);
16433 unsigned RegNo = FReg - RISCV::F0_F;
16434 unsigned HReg = RISCV::F0_H + RegNo;
16435 return std::make_pair(HReg, &RISCV::FPR16RegClass);
16442 .
Case(
"{v0}", RISCV::V0)
16443 .
Case(
"{v1}", RISCV::V1)
16444 .
Case(
"{v2}", RISCV::V2)
16445 .
Case(
"{v3}", RISCV::V3)
16446 .
Case(
"{v4}", RISCV::V4)
16447 .
Case(
"{v5}", RISCV::V5)
16448 .
Case(
"{v6}", RISCV::V6)
16449 .
Case(
"{v7}", RISCV::V7)
16450 .
Case(
"{v8}", RISCV::V8)
16451 .
Case(
"{v9}", RISCV::V9)
16452 .
Case(
"{v10}", RISCV::V10)
16453 .
Case(
"{v11}", RISCV::V11)
16454 .
Case(
"{v12}", RISCV::V12)
16455 .
Case(
"{v13}", RISCV::V13)
16456 .
Case(
"{v14}", RISCV::V14)
16457 .
Case(
"{v15}", RISCV::V15)
16458 .
Case(
"{v16}", RISCV::V16)
16459 .
Case(
"{v17}", RISCV::V17)
16460 .
Case(
"{v18}", RISCV::V18)
16461 .
Case(
"{v19}", RISCV::V19)
16462 .
Case(
"{v20}", RISCV::V20)
16463 .
Case(
"{v21}", RISCV::V21)
16464 .
Case(
"{v22}", RISCV::V22)
16465 .
Case(
"{v23}", RISCV::V23)
16466 .
Case(
"{v24}", RISCV::V24)
16467 .
Case(
"{v25}", RISCV::V25)
16468 .
Case(
"{v26}", RISCV::V26)
16469 .
Case(
"{v27}", RISCV::V27)
16470 .
Case(
"{v28}", RISCV::V28)
16471 .
Case(
"{v29}", RISCV::V29)
16472 .
Case(
"{v30}", RISCV::V30)
16473 .
Case(
"{v31}", RISCV::V31)
16475 if (VReg != RISCV::NoRegister) {
16476 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
16477 return std::make_pair(VReg, &RISCV::VMRegClass);
16478 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
16479 return std::make_pair(VReg, &RISCV::VRRegClass);
16480 for (
const auto *RC :
16481 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
16482 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
16483 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
16484 return std::make_pair(VReg, RC);
16490 std::pair<Register, const TargetRegisterClass *> Res =
16496 if (Res.second == &RISCV::GPRF16RegClass ||
16497 Res.second == &RISCV::GPRF32RegClass ||
16498 Res.second == &RISCV::GPRPF64RegClass)
16499 return std::make_pair(Res.first, &RISCV::GPRRegClass);
16507 if (ConstraintCode.
size() == 1) {
16508 switch (ConstraintCode[0]) {
16520 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
16523 if (Constraint.length() == 1) {
16524 switch (Constraint[0]) {
16552 GA->getValueType(0)));
16555 BA->getValueType(0)));
16568 if (Subtarget.hasStdExtZtso()) {
16570 return Builder.CreateFence(Ord);
16575 return Builder.CreateFence(Ord);
16584 if (Subtarget.hasStdExtZtso())
16589 if (Subtarget.enableSeqCstTrailingFence() &&
isa<StoreInst>(Inst) &&
16606 if (Subtarget.hasForcedAtomics())
16622 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
16624 return Intrinsic::riscv_masked_atomicrmw_add_i32;
16626 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
16628 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
16630 return Intrinsic::riscv_masked_atomicrmw_max_i32;
16632 return Intrinsic::riscv_masked_atomicrmw_min_i32;
16634 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
16636 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
16645 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
16647 return Intrinsic::riscv_masked_atomicrmw_add_i64;
16649 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
16651 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
16653 return Intrinsic::riscv_masked_atomicrmw_max_i64;
16655 return Intrinsic::riscv_masked_atomicrmw_min_i64;
16657 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
16659 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
16669 unsigned XLen = Subtarget.
getXLen();
16678 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
16679 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
16680 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
16693 unsigned ValWidth =
16696 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
16697 Result = Builder.CreateCall(LrwOpScwLoop,
16698 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
16701 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
16705 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
16713 if (Subtarget.hasForcedAtomics())
16725 unsigned XLen = Subtarget.
getXLen();
16726 Value *Ordering = Builder.getIntN(XLen,
static_cast<uint64_t>(Ord));
16727 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
16729 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
16730 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
16731 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
16732 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
16737 Value *Result = Builder.CreateCall(
16738 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
16740 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
16745 EVT DataVT)
const {
16758 return Subtarget.hasStdExtF();
16760 return Subtarget.hasStdExtD();
16792 "RVVBitsPerBlock changed, audit needed");
16802 if (!Subtarget.hasVendorXTHeadMemIdx())
16808 Base =
Op->getOperand(0);
16810 int64_t RHSC = RHS->getSExtValue();
16816 bool isLegalIndexedOffset =
false;
16817 for (
unsigned i = 0; i < 4; i++)
16818 if (
isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
16819 isLegalIndexedOffset =
true;
16823 if (!isLegalIndexedOffset)
16841 VT = LD->getMemoryVT();
16842 Ptr = LD->getBasePtr();
16844 VT = ST->getMemoryVT();
16845 Ptr = ST->getBasePtr();
16865 VT = LD->getMemoryVT();
16866 Ptr = LD->getBasePtr();
16868 VT = ST->getMemoryVT();
16869 Ptr = ST->getBasePtr();
16908 const Constant *PersonalityFn)
const {
16913 const Constant *PersonalityFn)
const {
16937 const bool HasExtMOrZmmul =
16938 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
16949 const APInt &Imm = ConstNode->getAPIntValue();
16950 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
16951 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
16955 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
16956 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
16957 (Imm - 8).isPowerOf2()))
16962 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
16963 ConstNode->hasOneUse()) {
16964 APInt ImmS = Imm.ashr(Imm.countr_zero());
16965 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
16966 (1 - ImmS).isPowerOf2())
16990 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
16999 unsigned *
Fast)
const {
17002 *
Fast = Subtarget.enableUnalignedScalarMem();
17003 return Subtarget.enableUnalignedScalarMem();
17019 *
Fast = Subtarget.enableUnalignedVectorMem();
17020 return Subtarget.enableUnalignedVectorMem();
17025 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
17026 bool IsABIRegCopy =
CC.has_value();
17028 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
17029 PartVT == MVT::f32) {
17032 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::i16, Val);
17036 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::f32, Val);
17047 if (PartVTBitSize % ValueVTBitSize == 0) {
17048 assert(PartVTBitSize >= ValueVTBitSize);
17055 if (ValueEltVT != PartEltVT) {
17056 if (PartVTBitSize > ValueVTBitSize) {
17058 assert(
Count != 0 &&
"The number of element should not be zero.");
17059 EVT SameEltTypeVT =
17065 Val = DAG.
getNode(ISD::BITCAST,
DL, PartVT, Val);
17080 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
17081 bool IsABIRegCopy =
CC.has_value();
17082 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
17083 PartVT == MVT::f32) {
17087 Val = DAG.
getNode(ISD::BITCAST,
DL, MVT::i32, Val);
17089 Val = DAG.
getNode(ISD::BITCAST,
DL, ValueVT, Val);
17100 if (PartVTBitSize % ValueVTBitSize == 0) {
17101 assert(PartVTBitSize >= ValueVTBitSize);
17102 EVT SameEltTypeVT = ValueVT;
17109 if (ValueEltVT != PartEltVT) {
17111 assert(
Count != 0 &&
"The number of element should not be zero.");
17114 Val = DAG.
getNode(ISD::BITCAST,
DL, SameEltTypeVT, Val);
17128 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
17135 unsigned Opc =
N->getOpcode();
17161 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
17180 if (FVTy->getNumElements() < 2)
17190 return Factor * LMUL <= 8;
17194 Align Alignment)
const {
17206 if (!Subtarget.enableUnalignedVectorMem() &&
17214 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
17215 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
17216 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
17217 Intrinsic::riscv_seg8_load};
17246 {VTy, LI->getPointerOperandType(), XLenTy});
17253 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
17254 Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
17255 Shuffles[i]->replaceAllUsesWith(SubVec);
17262 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
17263 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
17264 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
17265 Intrinsic::riscv_seg8_store};
17285 unsigned Factor)
const {
17290 ShuffleVTy->getNumElements() / Factor);
17292 SI->getPointerAddressSpace(),
17293 SI->getModule()->getDataLayout()))
17300 {VTy, SI->getPointerOperandType(), XLenTy});
17305 for (
unsigned i = 0; i < Factor; i++) {
17306 Value *Shuffle = Builder.CreateShuffleVector(
17315 Ops.
append({SI->getPointerOperand(), VL});
17317 Builder.CreateCall(VssegNFunc, Ops);
17328 if (DI->
getIntrinsicID() != Intrinsic::experimental_vector_deinterleave2)
17331 unsigned Factor = 2;
17349 {ResVTy, LI->getPointerOperandType(), XLenTy});
17353 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
17354 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
17355 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
17356 Intrinsic::riscv_vlseg8};
17366 Value *Vlseg = Builder.CreateCall(VlsegNFunc, Ops);
17378 if (II->
getIntrinsicID() != Intrinsic::experimental_vector_interleave2)
17381 unsigned Factor = 2;
17387 SI->getPointerAddressSpace(),
17388 SI->getModule()->getDataLayout()))
17398 {InVTy, SI->getPointerOperandType(), XLenTy});
17402 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
17403 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
17404 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
17405 Intrinsic::riscv_vsseg8};
17413 SI->getPointerOperand(), VL});
17423 "Invalid call instruction for a KCFI check");
17425 MBBI->getOpcode()));
17428 Target.setIsRenamable(
false);
17436#define GET_REGISTER_MATCHER
17437#include "RISCVGenAsmMatcher.inc"
17443 if (Reg == RISCV::NoRegister)
17445 if (Reg == RISCV::NoRegister)
17457 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
17459 if (NontemporalInfo ==
nullptr)
17467 int NontemporalLevel = 5;
17468 const MDNode *RISCVNontemporalInfo =
17469 I.getMetadata(
"riscv-nontemporal-domain");
17470 if (RISCVNontemporalInfo !=
nullptr)
17477 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
17478 "RISC-V target doesn't support this non-temporal domain.");
17480 NontemporalLevel -= 2;
17482 if (NontemporalLevel & 0b1)
17484 if (NontemporalLevel & 0b10)
17498 return TargetFlags;
17508#define GET_RISCVVIntrinsicsTable_IMPL
17509#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static unsigned MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static unsigned MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
Promote Memory to Register
This file provides utility analysis objects describing memory locations.
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal, SelectionDAG &DAG)
RISC-V doesn't have general instructions for integer setne/seteq, but we can check for equality with ...
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgVRs[]
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static SDValue getVLOp(uint64_t NumElts, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static MVT getLMUL1VT(MVT VT)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc, unsigned CVTFOpc)
static MachineBasicBlock * emitReadCycleWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM2s[]
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op)
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static std::optional< unsigned > getRoundModeIdx(const MachineInstr &MI)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue narrowIndex(SDValue N, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static MVT getMaskTypeFor(MVT VecVT)
Return the type of the mask type suitable for masking the provided vector type.
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
#define PseudoVFCVT_RM_CASE_M8(RMOpc, Opc)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISCV target specified VL op for a given SDNode.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo, std::optional< unsigned > FirstMaskArgument, CCState &State, const RISCVTargetLowering &TLI)
static SDValue performFADDSUB_VLCombine(SDNode *N, SelectionDAG &DAG)
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Combine a binary operation to its equivalent VW or VW_W form.
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
#define PseudoVFCVT_RM_CASE(RMOpc, Opc)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
#define PseudoVFCVT_RM_CASE_MF8(RMOpc, Opc)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRVVVPReductionOp(unsigned ISDOpcode)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performVFMUL_VLCombine(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static const MCPhysReg ArgVRM8s[]
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFCVT_RM(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static const Intrinsic::ID FixedVssegIntrIds[]
static bool isNonZeroAVL(const MachineOperand &MO)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
This class holds the attributes for a function, its return value, and its parameters.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
int64_t getLocMemOffset() const
unsigned getValNo() const
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
BasicBlock * GetInsertBlock() const
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineInstr - Allocate a new MachineInstr.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtZfhOrZfhminOrZhinxOrZhinxmin() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
bool hasStdExtZfhOrZfhmin() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
unsigned getRealMaxVLen() const
bool hasStdExtZhinxOrZhinxmin() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, bool &IsInc, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
int getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool shouldRemoveExtendFromGSIndex(EVT IndexVT, EVT DataVT) const override
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
const SDValue & getOperand(unsigned Num) const
bool isTargetStrictFPOpcode() const
Test if this node has a target-specific opcode that may raise FP exceptions (in the <target>ISD names...
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize Fixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
bool isStructTy() const
True if this is an instance of StructType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ADD
Simple integer binary arithmetic operators.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
static bool hasRoundModeOp(uint64_t TSFlags)
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static bool hasVecPolicyOp(uint64_t TSFlags)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
int getIntMatCost(const APInt &Val, unsigned Size, const FeatureBitset &ActiveFeatures, bool CompressionCost)
InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, std::optional< unsigned > FirstMaskArgument)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, std::optional< unsigned > FirstMaskArgument)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
initializer< Ty > init(const Ty &Val)
CodeModel::Model getCodeModel()
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
bool isReleaseOrStronger(AtomicOrdering AO)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
ArrayRef(const T &OneElt) -> ArrayRef< T >
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
void resetAll()
Resets the known state of all bits.
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)